Disable inactive club sites (#68)

Closes #51.

An API argument called `remove_inactive_club_reps` was added so that we can dynamically control whether we want to remove inactive club reps or not. The default action is only to disable club websites without changing group membership.

Reviewed-on: #68
Co-authored-by: Max Erenberg <merenber@csclub.uwaterloo.ca>
Co-committed-by: Max Erenberg <merenber@csclub.uwaterloo.ca>
pull/72/head
Max Erenberg 2 months ago committed by Raymond Li
parent 32b2dbb307
commit cfb5f77711
  1. 2
      .drone.yml
  2. 3
      .drone/auth1-setup.sh
  3. 4
      .drone/coffee-setup.sh
  4. 10
      .drone/common.sh
  5. 3
      .drone/mail-setup.sh
  6. 3
      .drone/phosphoric-acid-setup.sh
  7. 3
      README.md
  8. 2
      ceo/cli/entrypoint.py
  9. 37
      ceo/cli/webhosting.py
  10. 2
      ceo/utils.py
  11. 45
      ceo_common/interfaces/IClubWebHostingService.py
  12. 30
      ceo_common/interfaces/ILDAPService.py
  13. 12
      ceo_common/interfaces/IMailService.py
  14. 1
      ceo_common/interfaces/__init__.py
  15. 2
      ceo_common/model/HTTPClient.py
  16. 13
      ceod/api/app_factory.py
  17. 22
      ceod/api/webhosting.py
  18. 250
      ceod/model/ClubWebHostingService.py
  19. 34
      ceod/model/LDAPService.py
  20. 14
      ceod/model/MailService.py
  21. 1
      ceod/model/__init__.py
  22. 20
      ceod/model/templates/club_website_has_been_disabled.j2
  23. 2
      debian/control
  24. 2
      etc/ceo.ini
  25. 2
      etc/ceod.ini
  26. 3
      requirements.txt
  27. 62
      tests/ceo/cli/test_webhosting.py
  28. 1
      tests/ceo_dev.ini
  29. 43
      tests/ceod/api/test_webhosting.py
  30. 163
      tests/ceod/model/test_webhosting.py
  31. 2
      tests/ceod_dev.ini
  32. 1
      tests/ceod_test_local.ini
  33. 55
      tests/conftest.py
  34. 8
      tests/resources/apache2/disable-club.conf
  35. 11
      tests/utils.py

@ -10,7 +10,7 @@ steps:
# way to share system packages between steps
commands:
# install dependencies
- apt update && apt install -y libkrb5-dev libpq-dev python3-dev
- apt update && apt install -y libkrb5-dev libpq-dev python3-dev libaugeas0
- python3 -m venv venv
- . venv/bin/activate
- pip install -r dev-requirements.txt

@ -16,8 +16,6 @@ if [ -n "$CI" ]; then
rm /tmp/hosts
fi
export DEBIAN_FRONTEND=noninteractive
apt update
apt install -y psmisc
# If we don't do this then OpenLDAP uses a lot of RAM
@ -98,7 +96,6 @@ while true; do
fi
done
apt install -y netcat-openbsd
# sync with phosphoric-acid
nc -l 0.0.0.0 9000 &
if [ -z "$CI" ]; then

@ -8,9 +8,6 @@ set -ex
add_fqdn_to_hosts $(get_ip_addr $(hostname)) coffee
add_fqdn_to_hosts $(get_ip_addr auth1) auth1
export DEBIAN_FRONTEND=noninteractive
apt update
apt install --no-install-recommends -y default-mysql-server postgresql
# MYSQL
@ -49,7 +46,6 @@ REVOKE ALL ON SCHEMA public FROM public;
GRANT ALL ON SCHEMA public TO postgres;
EOF" postgres
apt install -y netcat-openbsd
if [ -z "$CI" ]; then
auth_setup coffee
fi

@ -23,6 +23,16 @@ cp .drone/k8s-authority.crt /etc/csc/k8s-authority.crt
# openssl is actually already present in the python Docker image,
# so we don't need to mock it out
# netcat is used for synchronization between the containers
export DEBIAN_FRONTEND=noninteractive
apt update
apt install -y netcat-openbsd
if [ "$(hostname)" != auth1 ]; then
# ceod uses Augeas, which is not installed by default in the Python
# Docker container
apt install -y libaugeas0
fi
get_ip_addr() {
getent hosts $1 | cut -d' ' -f1
}

@ -14,9 +14,6 @@ python -m tests.MockSMTPServer &
python -m tests.MockCloudStackServer &
python -m tests.MockHarborServer &
export DEBIAN_FRONTEND=noninteractive
apt update
apt install -y netcat-openbsd
auth_setup mail
# for the VHostManager

@ -13,9 +13,6 @@ if [ -z "$CI" ]; then
add_fqdn_to_hosts $(get_ip_addr mail) mail
fi
export DEBIAN_FRONTEND=noninteractive
apt update
apt install -y netcat-openbsd
auth_setup phosphoric-acid
# initialize the skel directory

@ -16,8 +16,7 @@ Docker containers instead, which are much easier to work with than the VM.
First, make sure you create the virtualenv:
```sh
docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster \
sh -c 'python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
```
Then bring up the containers:
```sh

@ -10,6 +10,7 @@ from .mailman import mailman
from .cloud import cloud
from .k8s import k8s
from .registry import registry
from .webhosting import webhosting
@click.group()
@ -27,3 +28,4 @@ cli.add_command(mailman)
cli.add_command(cloud)
cli.add_command(k8s)
cli.add_command(registry)
cli.add_command(webhosting)

@ -0,0 +1,37 @@
import click
from ..utils import http_post
from .utils import handle_sync_response
@click.group(short_help='Manage websites hosted by the main CSC web server')
def webhosting():
pass
@webhosting.command(short_help='Disable club sites with no active club reps')
@click.option('--dry-run', is_flag=True, default=False)
@click.option('--remove-inactive-club-reps', is_flag=True, default=False)
def disableclubsites(dry_run, remove_inactive_club_reps):
params = {}
if dry_run:
params['dry_run'] = 'true'
if remove_inactive_club_reps:
params['remove_inactive_club_reps'] = 'true'
if not dry_run:
click.confirm('Are you sure you want to disable the websites of clubs with no active club reps?', abort=True)
resp = http_post('/api/webhosting/disableclubsites', params=params)
disabled_club_names = handle_sync_response(resp)
if len(disabled_club_names) == 0:
if dry_run:
click.echo('No websites would have been disabled.')
else:
click.echo('No websites were disabled.')
else:
if dry_run:
click.echo('The following club websites would have been disabled:')
else:
click.echo('The following club websites were disabled:')
for club_name in disabled_club_names:
click.echo(club_name)

@ -21,6 +21,8 @@ def http_request(method: str, path: str, **kwargs) -> requests.Response:
host = cfg.get('ceod_mailman_host')
elif path.startswith('/api/cloud'):
host = cfg.get('ceod_cloud_host')
elif path.startswith('/api/webhosting'):
host = cfg.get('ceod_webhosting_host')
else:
host = cfg.get('ceod_admin_host')
return client.request(

@ -0,0 +1,45 @@
from typing import List
from zope.interface import Interface
class IClubWebHostingService(Interface):
"""
Performs operations on the configuration files of club websites hosted
by the CSC.
"""
def begin_transaction():
"""
Performs any steps necessary to query the website config files.
This must be called BEFORE any of the methods below, using a context
expression like so:
with club_site_mgr.begin_transaction():
club_site_mgr.disable_club_site('club1')
...
club_site_mgr.commit()
"""
def commit():
"""
Writes the config file changes to disk. This must be called at the
end of the context expression.
"""
def disable_club_site(club_name: str):
"""
Disables the site for the club. Note that commit() must still be
called to commit this change.
"""
def disable_sites_for_inactive_clubs(
dry_run: bool = False,
remove_inactive_club_reps: bool = False,
) -> List[str]:
"""
Disables sites for inactive clubs. If remove_inactive_club_reps is set
to True, then inactive club reps will be removed from club groups.
The list of clubs whose sites were disabled (or would have been
disabled, if dry_run is True) is returned.
"""

@ -9,10 +9,10 @@ from .IGroup import IGroup
class ILDAPService(Interface):
"""An interface to the LDAP database."""
def uid_to_dn(self, uid: str) -> str:
def uid_to_dn(uid: str) -> str:
"""Get the LDAP DN for the user with this UID."""
def group_cn_to_dn(self, cn: str) -> str:
def group_cn_to_dn(cn: str) -> str:
"""Get the LDAP DN for the group with this CN."""
def get_user(username: str) -> IUser:
@ -24,7 +24,7 @@ class ILDAPService(Interface):
Useful for displaying a list of users in a compact way.
"""
def get_users_with_positions(self) -> List[IUser]:
def get_users_with_positions() -> List[IUser]:
"""Retrieve users who have a non-empty position attribute."""
def add_user(user: IUser):
@ -36,7 +36,7 @@ class ILDAPService(Interface):
def remove_user(user: IUser):
"""Remove this user from the database."""
def get_group(cn: str, is_club: bool = False) -> IGroup:
def get_group(cn: str) -> IGroup:
"""Retrieve the group with the given cn (Unix group name)."""
def add_group(group: IGroup):
@ -88,15 +88,33 @@ class ILDAPService(Interface):
described above.
"""
def get_nonflagged_expired_users(self) -> List[IUser]:
def get_nonflagged_expired_users() -> List[IUser]:
"""
Retrieves members whose term or nonMemberTerm does not contain the
current or the last term.
"""
def get_expiring_users(self) -> List[IUser]:
def get_expiring_users() -> List[IUser]:
"""
Retrieves members whose membership will expire in less than a month.
This is used to send membership renewal reminders at the beginning
of a term, during the one-month grace period.
"""
def get_clubs() -> List[IGroup]:
"""
Retrieves all clubs.
"""
# couldn't import the Term class from ceo_common.model due to some
# circular import issue...
def get_club_reps_non_member_terms(club_reps: List[str]) -> Dict[str, List['Term']]: # noqa: F821
"""
Retrieves the non-member terms for the given club reps.
e.g.
{
"user1": [w2022, s2022],
"user2": [s2022],
...
}
"""

@ -24,20 +24,26 @@ class IMailService(Interface):
during the transaction.
"""
def send_membership_renewal_reminder(self, user: IUser):
def send_membership_renewal_reminder(user: IUser):
"""
Send a reminder to the user that their membership will expire
soon.
"""
def send_cloud_account_will_be_deleted_message(self, user: IUser):
def send_cloud_account_will_be_deleted_message(user: IUser):
"""
Send a warning message to the user that their cloud resources
will be deleted if they do not renew their membership.
"""
def send_cloud_account_has_been_deleted_message(self, user: IUser):
def send_cloud_account_has_been_deleted_message(user: IUser):
"""
Send a message to the user that their cloud resources have
been deleted.
"""
def send_club_website_has_been_disabled_message(club: str, address: str):
"""
Send a message to the club stating that their website has been
disabled.
"""

@ -14,3 +14,4 @@ from .IDatabaseService import IDatabaseService
from .IVHostManager import IVHostManager
from .IKubernetesService import IKubernetesService
from .IContainerRegistryService import IContainerRegistryService
from .IClubWebHostingService import IClubWebHostingService

@ -43,7 +43,7 @@ class HTTPClient:
if flask.has_request_context() and 'client_token' in g:
# This is reached when we are the server and the client has
# forwarded their credentials to us.
spnego_kwargs['creds'] = gssapi.Credentials(token=flask.g.client_token)
spnego_kwargs['creds'] = gssapi.Credentials(token=g.client_token)
elif delegate:
# This is reached when we are the client and we want to
# forward our credentials to the server.

@ -9,13 +9,13 @@ from .error_handlers import register_error_handlers
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \
ICloudStackService, ICloudResourceManager, IKubernetesService, IVHostManager, \
IContainerRegistryService
IContainerRegistryService, IClubWebHostingService
from ceo_common.model import Config, HTTPClient, RemoteMailmanService
from ceod.api.spnego import init_spnego
from ceod.model import KerberosService, LDAPService, FileService, \
MailmanService, MailService, UWLDAPService, CloudStackService, \
CloudResourceManager, KubernetesService, VHostManager, \
ContainerRegistryService
ContainerRegistryService, ClubWebHostingService
from ceod.db import MySQLService, PostgreSQLService
@ -49,6 +49,10 @@ def create_app(flask_config={}):
from ceod.api import cloud
app.register_blueprint(cloud.bp, url_prefix='/api/cloud')
if hostname == cfg.get('ceod_webhosting_host'):
from ceod.api import webhosting
app.register_blueprint(webhosting.bp, url_prefix='/api/webhosting')
from ceod.api import groups
app.register_blueprint(groups.bp, url_prefix='/api/groups')
@ -112,6 +116,11 @@ def register_services(app):
uwldap_srv = UWLDAPService()
component.provideUtility(uwldap_srv, IUWLDAPService)
# ClubWebHostingService
if hostname == cfg.get('ceod_webhosting_host'):
webhosting_srv = ClubWebHostingService()
component.provideUtility(webhosting_srv, IClubWebHostingService)
# MySQLService, PostgreSQLService
if hostname == cfg.get('ceod_database_host'):
mysql_srv = MySQLService()

@ -0,0 +1,22 @@
from flask import Blueprint, request
from flask.json import jsonify
from zope import component
from .utils import authz_restrict_to_syscom, is_truthy
from ceo_common.interfaces import IClubWebHostingService
bp = Blueprint('webhosting', __name__)
@bp.route('/disableclubsites', methods=['POST'])
@authz_restrict_to_syscom
def expire_club_sites():
dry_run = is_truthy(request.args.get('dry_run', 'false'))
remove_inactive_club_reps = is_truthy(request.args.get('remove_inactive_club_reps', 'false'))
webhosting_srv = component.getUtility(IClubWebHostingService)
disabled_sites = webhosting_srv.disable_sites_for_inactive_clubs(
dry_run=dry_run,
remove_inactive_club_reps=remove_inactive_club_reps,
)
return jsonify(disabled_sites)

@ -0,0 +1,250 @@
from collections import defaultdict
import contextlib
import glob
import os
import re
import subprocess
from threading import Lock
import traceback
from typing import List
from augeas import Augeas
from zope import component
from zope.interface import implementer
from ceo_common.interfaces import ILDAPService, IMailService, \
IClubWebHostingService, IConfig
from ceo_common.logger_factory import logger_factory
from ceo_common.model import Term
# NOTE: We are assuming that the name of a club is the same as its home
# directory (under /users). While this rule is not enforced, it has been true
# up until now.
# FIXME: don't assume this
APACHE_USERDIR_RE = re.compile(r'^/users/(?P<club_name>[0-9a-z-]+)/www/?$')
# This is where all the <Directory> directives for disabled clubs are stored.
APACHE_DISABLED_CLUBS_FILE = 'conf-available/disable-club.conf'
# This is the file which contains the snippet to disable a single club.
APACHE_DISABLING_SNIPPET_FILE = 'snippets/disable-club.conf'
# This is the maximum number of consecutive terms for which a club is allowed
# to have no active club reps. After this many terms have passed, the club's
# website may be disabled.
MAX_TERMS_WITH_NO_ACTIVE_CLUB_REPS = 3
logger = logger_factory(__name__)
def is_a_disabling_snippet(snippet_path: str) -> bool:
return snippet_path.startswith('snippets/disable-')
@implementer(IClubWebHostingService)
class ClubWebHostingService:
def __init__(self, augeas_root='/'):
cfg = component.getUtility(IConfig)
self.aug_root = augeas_root
self.apache_dir = os.path.join(augeas_root, 'etc/apache2')
self.sites_available_dir = os.path.join(self.apache_dir, 'sites-available')
self.conf_available_dir = os.path.join(self.apache_dir, 'conf-available')
self.clubs_home = cfg.get('clubs_home')
self.aug = None
self.clubs = None
self.made_at_least_one_change = False
self.lock = Lock()
@contextlib.contextmanager
def _hold_lock(self):
try:
self.lock.acquire()
yield
finally:
self.lock.release()
@contextlib.contextmanager
def begin_transaction(self):
with self._hold_lock():
try:
self.aug = Augeas(self.aug_root)
self.clubs = defaultdict(lambda: {'disabled': False, 'email': None})
self._get_club_emails()
self._get_disabled_sites()
yield
finally:
if self.aug is not None:
self.aug.close()
self.aug = None
self.clubs = None
self.made_at_least_one_change = False
def _run(self, args: List[str], **kwargs):
subprocess.run(args, check=True, **kwargs)
def _reload_web_server(self):
logger.debug('Reloading Apache')
self._run(['systemctl', 'reload', 'apache2'])
# This requires the APACHE_CONFIG_CRON environment variable to be
# set to 1 (e.g. in a systemd drop-in)
# See /etc/apache2/.git/hooks/pre-commit on caffeine
def _git_commit(self):
if not os.path.isdir(os.path.join(self.apache_dir, '.git')):
logger.debug('No git folder found in Apache directory')
return
logger.debug('Committing changes to git repository')
self._run(['git', 'add', APACHE_DISABLED_CLUBS_FILE], cwd=self.apache_dir)
self._run(['git', 'commit', '-m', '[ceo] disable club websites'], cwd=self.apache_dir)
def commit(self):
if not self.made_at_least_one_change:
return
logger.debug('Saving Augeas changes')
self.aug.save()
self._reload_web_server()
self._git_commit()
def _get_club_emails(self):
config_file_paths = glob.glob(self.sites_available_dir + '/club-*.conf')
for config_file_path in config_file_paths:
club_name = None
club_email = None
filename = os.path.basename(config_file_path)
directive_paths = self.aug.match(f'/files/etc/apache2/sites-available/{filename}/VirtualHost/directive')
for directive_path in directive_paths:
directive = self.aug.get(directive_path)
directive_value = self.aug.get(directive_path + '/arg')
if directive == 'DocumentRoot':
match = APACHE_USERDIR_RE.match(directive_value)
if match is not None:
club_name = match.group('club_name')
elif directive == 'ServerAdmin':
club_email = directive_value
if club_name is not None:
self.clubs[club_name]['email'] = club_email
def _get_disabled_sites(self):
config_file_paths = glob.glob(self.conf_available_dir + '/disable-*.conf')
for config_file_path in config_file_paths:
filename = os.path.basename(config_file_path)
directory_paths = self.aug.match(f'/files/etc/apache2/conf-available/{filename}/Directory')
for directory_path in directory_paths:
directory = self.aug.get(directory_path + '/arg')
match = APACHE_USERDIR_RE.match(directory)
if match is None:
continue
club_name = match.group('club_name')
directive_paths = self.aug.match(directory_path + '/directive')
for directive_path in directive_paths:
if self.aug.get(directive_path) != 'Include':
continue
included_file = self.aug.get(directive_path + '/arg')
if is_a_disabling_snippet(included_file):
self.clubs[club_name]['disabled'] = True
def disable_club_site(self, club_name: str):
logger.info(f'Disabling website for {club_name}')
directory_path = f'/files/etc/apache2/{APACHE_DISABLED_CLUBS_FILE}/Directory'
num_directories = len(self.aug.match(directory_path))
# Create a new <Directory> section
directory_path += '[%d]' % (num_directories + 1)
# FIXME: use self.clubs_home here instead (need to update unit tests)
self.aug.set(directory_path + '/arg', f'/users/{club_name}/www')
self.aug.set(directory_path + '/directive', 'Include')
self.aug.set(directory_path + '/directive/arg', APACHE_DISABLING_SNIPPET_FILE)
self.clubs[club_name]['disabled'] = True
self.made_at_least_one_change = True
def _site_uses_php(self, club_name: str) -> bool:
www = f'{self.clubs_home}/{club_name}/www'
if os.path.isdir(www):
# We're just going to look one level deep; that should be good enough.
for filename in os.listdir(www):
filepath = os.path.join(www, filename)
if os.path.isfile(filepath) and filename.endswith('.php'):
return True
return False
# This method needs to be called from within a transaction (uses self.clubs)
def _need_to_disable_inactive_club_site(self, club_name: str) -> bool:
if self.clubs[club_name]['disabled']:
# already disabled - nothing to do
return False
if not self._site_uses_php(club_name):
# These are the only sites which are actually a security concern
# to us - it's OK for a purely static website to be unmaintained.
return False
return True
def disable_sites_for_inactive_clubs(
self,
dry_run: bool = False,
remove_inactive_club_reps: bool = False,
) -> List[str]:
ldap_srv = component.getUtility(ILDAPService)
mail_srv = component.getUtility(IMailService)
all_clubs = ldap_srv.get_clubs()
club_rep_uids = list({
member
for club in all_clubs
for member in club.members
})
club_reps = ldap_srv.get_club_reps_non_member_terms(club_rep_uids)
# If a club rep's last non-member term is before cutoff_term, then
# they are considered inactive
cutoff_term = Term.current() - MAX_TERMS_WITH_NO_ACTIVE_CLUB_REPS
active_club_reps = {
club_rep
for club_rep, non_member_terms in club_reps.items()
if len(non_member_terms) > 0 and max(non_member_terms) >= cutoff_term
}
# a club is inactive if it does not have at least one active club rep
inactive_clubs = [
club
for club in all_clubs
if not any(map(lambda member: member in active_club_reps, club.members))
]
# STEP 1: update the Apache configs
with self.begin_transaction():
clubs_to_disable = [
club.cn
for club in inactive_clubs
if self._need_to_disable_inactive_club_site(club.cn)
]
if dry_run:
return clubs_to_disable
for club_name in clubs_to_disable:
self.disable_club_site(club_name)
self.commit()
# self.clubs is set to None once the transaction closes, so make a copy now
clubs_info = self.clubs.copy()
# STEP 2: send emails to clubs whose websites were disabled
clubs_who_were_not_notified = set()
for club_name in clubs_to_disable:
address = clubs_info['email']
if address is None:
clubs_who_were_not_notified.add(club_name)
continue
try:
mail_srv.send_club_website_has_been_disabled_message(club_name, address)
except Exception:
trace = traceback.format_exc()
logger.error(f'Failed to send email to {address}:\n{trace}')
clubs_who_were_not_notified.add(club_name)
# STEP 3: remove inactive club reps from Unix groups
if remove_inactive_club_reps:
for club in all_clubs:
if club.cn in clubs_who_were_not_notified:
continue
# club.members gets modified after calling club.remove_member(),
# so we need to make a copy
members = club.members.copy()
for member in members:
if member not in active_club_reps:
logger.info(f'Removing {member} from {club.cn}')
club.remove_member(member)
return clubs_to_disable

@ -344,3 +344,37 @@ class LDAPService:
conn.modify(self.uid_to_dn(uid), changes)
return users_to_change
def _get_club_uids(self, conn: ldap3.Connection) -> List[str]:
conn.search(self.ldap_users_base, '(objectClass=club)', attributes=['uid'])
return [entry.uid.value for entry in conn.entries]
def get_clubs(self) -> List[IGroup]:
batch_size = 100
conn = self._get_ldap_conn()
club_uids = self._get_club_uids(conn)
clubs = []
for i in range(0, len(club_uids), batch_size):
club_uids_slice = club_uids[i:i + batch_size]
filter = '(|' + ''.join([f'(cn={uid})' for uid in club_uids_slice]) + ')'
conn.search(self.ldap_groups_base, filter, attributes=ldap3.ALL_ATTRIBUTES)
for entry in conn.entries:
clubs.append(Group.deserialize_from_ldap(entry))
return clubs
def get_club_reps_non_member_terms(self, club_reps: List[str]) -> Dict[str, List[Term]]:
batch_size = 100
conn = self._get_ldap_conn()
club_reps_terms = {}
for i in range(0, len(club_reps), batch_size):
club_reps_slice = club_reps[i:i + batch_size]
filter = '(|' + ''.join([f'(uid={uid})' for uid in club_reps_slice]) + ')'
conn.search(self.ldap_users_base, filter, attributes=['uid', 'nonMemberTerm'])
for entry in conn.entries:
uid = entry.uid.value
if 'nonMemberTerm' in entry.entry_attributes:
non_member_terms = list(map(Term, entry.nonMemberTerm.values))
else:
non_member_terms = []
club_reps_terms[uid] = non_member_terms
return club_reps_terms

@ -137,3 +137,17 @@ class MailService:
},
body,
)
def send_club_website_has_been_disabled_message(self, club: str, address: str):
template = self.jinja_env.get_template('club_website_has_been_disabled.j2')
body = template.render(club=club)
self.send(
f'Computer Science Club <ceo+webhosting@{self.base_domain}>',
f'{club} <{address}>',
{
'Subject': 'Your club website has been disabled',
'Cc': f'root@{self.base_domain}',
'Reply-To': f'syscom@{self.base_domain}',
},
body,
)

@ -13,3 +13,4 @@ from .MailmanService import MailmanService
from .VHostManager import VHostManager
from .KubernetesService import KubernetesService
from .ContainerRegistryService import ContainerRegistryService
from .ClubWebHostingService import ClubWebHostingService

@ -0,0 +1,20 @@
Hello {{ club }} representatives,
This is an automated message from ceo, the CSC Electronic Office.
Since your club has not had any active CSC club reps for 3 or more consecutive
terms, your club's website has been disabled. Websites require active
maintenance and software upgrades, and an unmaintained website poses a
security risk to CSC and to the University of Waterloo.
If you wish for your site to be re-enabled, please register at least
one club rep with CSC and, if necessary, upgrade the software running
your website. A club rep membership is free. You can find instructions
on our website for how to obtain a membership:
https://csclub.uwaterloo.ca/get-involved/
If you have any questions or concerns, please contact syscom@csclub.uwaterloo.ca.
Sincerely,
ceo

2
debian/control vendored

@ -11,6 +11,7 @@ Build-Depends: debhelper (>= 12.1.1),
python3-venv (>= 3.7),
libkrb5-dev (>= 1.17),
libpq-dev (>= 11.13),
libaugeas0 (>= 1.11),
scdoc (>= 1.9)
Package: ceo-common
@ -19,6 +20,7 @@ Depends: python3 (>= 3.7),
krb5-user (>= 1.17),
libkrb5-3 (>= 1.17),
libpq5 (>= 11.13),
libaugeas0 (>= 1.11),
${python3:Depends},
${misc:Depends}
Description: CSC Electronic Office common files

@ -7,6 +7,8 @@ uw_domain = uwaterloo.ca
admin_host = phosphoric-acid
# this is the host with root access to the databases
database_host = caffeine
# this is the host where clubs' websites are hosted
webhosting_host = caffeine
# this is the host which can make API requests to Mailman
mailman_host = mailman
# this is the host running a CloudStack management server

@ -8,6 +8,8 @@ admin_host = phosphoric-acid
fs_root_host = phosphoric-acid
# this is the host with root access to the databases
database_host = caffeine
# this is the host where clubs' websites are hosted
webhosting_host = caffeine
# this is the host which can make API requests to Mailman
mailman_host = mailman
# this is the host which is running a CloudStack management server

@ -7,9 +7,10 @@ Jinja2==3.1.2
ldap3==2.9.1
mysql-connector-python==8.0.26
psycopg2==2.9.1
python-augeas==1.1.0
requests==2.26.0
requests-gssapi==1.2.3
urwid==2.1.2
werkzeug==2.1.2
Werkzeug==2.1.2
zope.component==5.0.1
zope.interface==5.4.0

@ -0,0 +1,62 @@
from click.testing import CliRunner
from ceo.cli import cli
from ceo_common.model import Term
from tests.utils import (
create_php_file_for_club,
reset_disable_club_conf,
set_datetime_in_app_process,
restore_datetime_in_app_process,
)
def test_disable_club_sites(
cfg, cli_setup, app_process, webhosting_srv, webhosting_srv_resources,
new_club_gen, new_user_gen, g_admin_ctx, ldap_srv_session,
):
runner = CliRunner()
term = Term.current()
clubs_home = cfg.get('clubs_home')
with new_club_gen() as group, new_user_gen() as user:
create_php_file_for_club(clubs_home, group.cn)
user.add_non_member_terms([str(Term.current())])
group.add_member(user.uid)
set_datetime_in_app_process(app_process, (term + 4).to_datetime())
result = runner.invoke(cli, ['webhosting', 'disableclubsites', '--dry-run'])
assert result.exit_code == 0
assert result.output.endswith(
'The following club websites would have been disabled:\n'
f'{group.cn}\n'
)
result = runner.invoke(cli, ['webhosting', 'disableclubsites'], input='y\n')
assert result.exit_code == 0
assert result.output.endswith(
'The following club websites were disabled:\n'
f'{group.cn}\n'
)
with g_admin_ctx():
group = ldap_srv_session.get_group(group.cn)
assert group.members == [user.uid]
reset_disable_club_conf(webhosting_srv)
result = runner.invoke(
cli,
['webhosting', 'disableclubsites', '--remove-inactive-club-reps'],
input='y\n'
)
assert result.exit_code == 0
assert result.output.endswith(
'The following club websites were disabled:\n'
f'{group.cn}\n'
)
with g_admin_ctx():
group = ldap_srv_session.get_group(group.cn)
assert group.members == []
restore_datetime_in_app_process(app_process)

@ -6,6 +6,7 @@ uw_domain = uwaterloo.internal
# this is the host with the ceod/admin Kerberos key
admin_host = phosphoric-acid
database_host = coffee
webhosting_host = coffee
mailman_host = mail
cloud_host = mail
use_https = false

@ -0,0 +1,43 @@
import datetime
from unittest.mock import patch
from ceo_common.model import Term
import ceo_common.utils
from tests.utils import create_php_file_for_club, reset_disable_club_conf
def test_disable_club_sites(
cfg, client, webhosting_srv, webhosting_srv_resources, new_club_gen,
new_user_gen, g_admin_ctx, ldap_srv_session,
):
term = Term.current()
clubs_home = cfg.get('clubs_home')
with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock:
now_mock.return_value = datetime.datetime.now()
with new_club_gen() as group, new_user_gen() as user:
create_php_file_for_club(clubs_home, group.cn)
user.add_non_member_terms([str(Term.current())])
group.add_member(user.uid)
now_mock.return_value = (term + 4).to_datetime()
status, data = client.post('/api/webhosting/disableclubsites?dry_run=true')
assert status == 200
assert data == [group.cn]
status, data = client.post('/api/webhosting/disableclubsites')
assert status == 200
assert data == [group.cn]
with g_admin_ctx():
group = ldap_srv_session.get_group(group.cn)
assert group.members == [user.uid]
reset_disable_club_conf(webhosting_srv)
status, data = client.post('/api/webhosting/disableclubsites?remove_inactive_club_reps=true')
assert status == 200
assert data == [group.cn]
with g_admin_ctx():
group = ldap_srv_session.get_group(group.cn)
assert group.members == []

@ -0,0 +1,163 @@
import datetime
import os
import re
import subprocess
from unittest.mock import patch
from ceo_common.model import Term
import ceo_common.utils
from tests.utils import create_php_file_for_club
def create_website_config_for_club(sites_available_dir, cn, filename=None):
if filename is None:
filename = f'club-{cn}.conf'
filepath = os.path.join(sites_available_dir, filename)
with open(filepath, 'w') as fo:
fo.write(f"""
<VirtualHost *:80>
ServerName {cn}.uwaterloo.internal
ServerAdmin {cn}@{cn}.uwaterloo.internal
DocumentRoot /users/{cn}/www/
</VirtualHost>
""")
def get_enabled_sites(webhosting_srv):
return [
club_name
for club_name, club_info in webhosting_srv.clubs.items()
if not club_info['disabled']
]
def test_disable_club_sites(webhosting_srv, webhosting_srv_resources, new_club_gen):
sites_available_dir = webhosting_srv.sites_available_dir
with new_club_gen() as group1, new_club_gen() as group2:
create_website_config_for_club(sites_available_dir, group1.cn)
create_website_config_for_club(sites_available_dir, group2.cn, 'club-somerandomname.conf')
with webhosting_srv.begin_transaction():
enabled_clubs = get_enabled_sites(webhosting_srv)
assert sorted(enabled_clubs) == [group1.cn, group2.cn]
webhosting_srv.disable_club_site(group1.cn)
# Make sure that if we don't call commit(), nothing gets saved
with webhosting_srv.begin_transaction():
enabled_clubs = get_enabled_sites(webhosting_srv)
assert sorted(enabled_clubs) == [group1.cn, group2.cn]
webhosting_srv.disable_club_site(group1.cn)
webhosting_srv.commit()
# Now that we committed the changes, they should be persistent
with webhosting_srv.begin_transaction():
enabled_clubs = get_enabled_sites(webhosting_srv)
assert enabled_clubs == [group2.cn]
def test_disable_inactive_club_sites(
cfg, webhosting_srv, webhosting_srv_resources, g_admin_ctx, new_club_gen,
new_user_gen, mock_mail_server,
):
sites_available_dir = webhosting_srv.sites_available_dir
term = Term.current()
clubs_home = cfg.get('clubs_home')
with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock:
now_mock.return_value = datetime.datetime.now()
with new_club_gen() as group1, \
new_club_gen() as group2, \
new_user_gen() as user1, \
new_user_gen() as user2:
create_website_config_for_club(sites_available_dir, group1.cn)
create_website_config_for_club(sites_available_dir, group2.cn)
create_php_file_for_club(clubs_home, group1.cn)
with g_admin_ctx():
# group1 has no club reps so it should be disabled
# group2 has no club reps but it doesn't use PHP
assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [group1.cn]
user1.add_non_member_terms([str(Term.current())])
group1.add_member(user1.uid)
with g_admin_ctx():
# group1 has an active club rep, so it shouldn't be disabled anymore
assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
now_mock.return_value = (term + 3).to_datetime()
with g_admin_ctx():
# club reps are allowed to be inactive for up to 3 terms
assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
now_mock.return_value = (term + 4).to_datetime()
with g_admin_ctx():
# club site should be disabled now that club rep is inactive
assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == [group1.cn]
user2.add_non_member_terms([str(Term.current())])
group1.add_member(user2.uid)
with g_admin_ctx():
# group1 has a new club rep, so it shouldn't be disabled anymore
assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
create_php_file_for_club(clubs_home, group2.cn)
group2.add_member(user2.uid)
with g_admin_ctx():
# user2 is an active club rep for both group1 and group2
assert webhosting_srv.disable_sites_for_inactive_clubs(dry_run=True) == []
now_mock.return_value = (term + 8).to_datetime()
mock_mail_server.messages.clear()
subprocess.run(['git', 'init'], cwd=webhosting_srv.apache_dir, check=True)
with g_admin_ctx():
disabled_sites = webhosting_srv.disable_sites_for_inactive_clubs()
# user2 expired and both sites use PHP, so they should both be disabled
assert sorted(disabled_sites) == [group1.cn, group2.cn]
# since each club had a ServerAdmin directive, they should both have received
# notification emails
assert len(mock_mail_server.messages) == 2
with open(webhosting_srv.conf_available_dir + '/disable-club.conf') as fi:
disable_club_conf_content = fi.read()
print(disable_club_conf_content)
for group in [group1, group2]:
pat = re.compile(
(
rf'^<Directory "?/users/{group.cn}/www"?>\n'
r'\s*Include snippets/disable-club\.conf\n'
r'</Directory>$'
),
re.MULTILINE
)
assert pat.search(disable_club_conf_content) is not None
with g_admin_ctx():
# Club sites should only be disabled once
assert webhosting_srv.disable_sites_for_inactive_clubs() == []
mock_mail_server.messages.clear()
def test_remove_inactive_club_reps(
cfg, webhosting_srv, webhosting_srv_resources, g_admin_ctx, new_club_gen,
new_user_gen, ldap_srv_session,
):
term = Term.current()
clubs_home = cfg.get('clubs_home')
with patch.object(ceo_common.utils, 'get_current_datetime') as now_mock:
now_mock.return_value = datetime.datetime.now()
with new_club_gen() as group, \
new_user_gen() as user1, \
new_user_gen() as user2:
create_php_file_for_club(clubs_home, group.cn)
for user in [user1, user2]:
user.add_non_member_terms([str(Term.current())])
group.add_member(user.uid)
now_mock.return_value = (term + 4).to_datetime()
with g_admin_ctx():
webhosting_srv.disable_sites_for_inactive_clubs(remove_inactive_club_reps=True)
group = ldap_srv_session.get_group(group.cn)
assert group.members == []
# Make sure that inactive club reps get removed even if the site
# has already been disabled
group.add_member(user1.uid)
with g_admin_ctx():
webhosting_srv.disable_sites_for_inactive_clubs(remove_inactive_club_reps=True)
group = ldap_srv_session.get_group(group.cn)
assert group.members == []

@ -8,6 +8,8 @@ admin_host = phosphoric-acid
fs_root_host = phosphoric-acid
mailman_host = mail
database_host = coffee
# this is the host where clubs' websites are hosted
webhosting_host = coffee
cloud_host = mail
use_https = false
port = 9987

@ -9,6 +9,7 @@ fs_root_host = phosphoric-acid
mailman_host = phosphoric-acid
database_host = phosphoric-acid
cloud_host = phosphoric-acid
webhosting_host = phosphoric-acid
use_https = false
port = 9988

@ -7,6 +7,7 @@ import shutil
import subprocess
from subprocess import DEVNULL
import sys
import tempfile
import threading
import time
from unittest.mock import Mock
@ -24,11 +25,12 @@ from .utils import ( # noqa: F401
gssapi_token_ctx,
ccache_cleanup,
mocks_for_create_user_ctx,
reset_disable_club_conf,
)
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
IDatabaseService, ICloudStackService, IKubernetesService, IVHostManager, \
ICloudResourceManager, IContainerRegistryService
ICloudResourceManager, IContainerRegistryService, IClubWebHostingService
from ceo_common.model import Config, HTTPClient, Term
import ceo_common.utils
from ceod.api import create_app
@ -36,7 +38,7 @@ from ceod.db import MySQLService, PostgreSQLService
from ceod.model import KerberosService, LDAPService, FileService, User, \
MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \
CloudStackService, KubernetesService, VHostManager, CloudResourceManager, \
ContainerRegistryService
ContainerRegistryService, ClubWebHostingService
from .MockSMTPServer import MockSMTPServer
from .MockMailmanServer import MockMailmanServer
from .MockCloudStackServer import MockCloudStackServer
@ -132,12 +134,12 @@ def g_syscom(app):
"""
with gssapi_token_ctx('ctdalek') as token, app.app_context():
try:
flask.g.sasl_user = 'ctdalek'
flask.g.auth_user = 'ctdalek'
flask.g.client_token = token
yield
finally:
flask.g.pop('client_token')
flask.g.pop('sasl_user')
flask.g.pop('auth_user')
@pytest.fixture(scope='session')
@ -237,6 +239,14 @@ def uwldap_srv(cfg, ldap_conn):
delete_subtree(conn, base_dn)
@pytest.fixture(scope='session')
def webhosting_srv():
with tempfile.TemporaryDirectory() as tmpdir:
srv = ClubWebHostingService(tmpdir)
component.getGlobalSiteManager().registerUtility(srv, IClubWebHostingService)
yield srv
@pytest.fixture(scope='session')
def mock_mail_server():
mock_server = MockSMTPServer()
@ -343,6 +353,7 @@ def app(
k8s_srv,
registry_srv,
cloud_mgr,
webhosting_srv,
):
app = create_app({'TESTING': True})
return app
@ -442,6 +453,31 @@ def new_user(new_user_gen):
yield user
@pytest.fixture
def new_club_gen(client, g_admin_ctx, ldap_srv_session): # noqa: F811
new_club_id_counter = 31001
@contextlib.contextmanager
def wrapper():
nonlocal new_club_id_counter
cn = 'test' + str(new_club_id_counter)
new_club_id_counter += 1
status, data = client.post('/api/groups', json={
'cn': cn,
'description': 'Test ' + str(new_club_id_counter),
})
assert status == 200
assert data[-1]['status'] == 'completed'
with g_admin_ctx():
group = ldap_srv_session.get_group(cn)
yield group
status, data = client.delete(f'/api/groups/{cn}')
assert status == 200
assert data[-1]['status'] == 'completed'
return wrapper
@pytest.fixture
def simple_group():
return Group(
@ -507,6 +543,17 @@ def uwldap_user(cfg, uwldap_srv, ldap_conn):
conn.delete(dn)
@pytest.fixture
def webhosting_srv_resources(webhosting_srv):
os.makedirs(webhosting_srv.conf_available_dir, exist_ok=True)
os.makedirs(webhosting_srv.sites_available_dir, exist_ok=True)
reset_disable_club_conf(webhosting_srv)
for file in os.listdir(webhosting_srv.sites_available_dir):
os.unlink(os.path.join(webhosting_srv.sites_available_dir, file))
if os.path.isdir(webhosting_srv.apache_dir + '/.git'):
shutil.rmtree(webhosting_srv.apache_dir + '/.git')
@pytest.fixture(scope='module')
def app_process(cfg, app, http_client):
port = cfg.get('ceod_port')

@ -0,0 +1,8 @@
# This file would normally be in /etc/apache2/conf-available/disable-club.conf
# on caffeine.
Alias /~sysadmin/disabled /users/sysadmin/www/disabled
<Directory "/users/somerandomclub/www">
Include snippets/disable-club.conf
</Directory>

@ -1,6 +1,7 @@
import ceod.utils as ceod_utils
import contextlib
import os
import shutil
import subprocess
from subprocess import DEVNULL
import tempfile
@ -70,3 +71,13 @@ def set_datetime_in_app_process(app_process, value):
def restore_datetime_in_app_process(app_process):
set_datetime_in_app_process(app_process, None)
def reset_disable_club_conf(webhosting_srv):
shutil.copy('tests/resources/apache2/disable-club.conf', webhosting_srv.conf_available_dir)
def create_php_file