Compare commits

...

24 Commits

Author SHA1 Message Date
Jonathan Leung b507c56136 Show groups in member for API, CLI and TUI (#82) 1 week ago
Max Erenberg c0c9736593 Use the admin creds in the HTTPClient when necessary (#85) 1 month ago
Max Erenberg 1e452d10ce Assume program is Alumni if UWLDAP is missing data (#84) 1 month ago
Max Erenberg 6a1fa81b82 merenber signs the packages 1 month ago
Max Erenberg 6df1f4d459 Revert "Simplify packaging" 1 month ago
Edwin 2cf9e25b59 More fixes 1 month ago
Edwin 9ff3d850c9 Release 1.0.24 1 month ago
Edwin b4a1373559 Simplify packaging 1 month ago
Raymond Li dceb5d6572
Revert "#63: Add positions to CEO (#79)" 2 months ago
Jonathan Leung c30ca54752 Sort group member listing by WatIAM ID (#78) 2 months ago
Nathan Chung 3b7c89c925 #63: Add positions to CEO (#79) 2 months ago
Max Erenberg a2324090f3 update README instructions for curl 2 months ago
Max Erenberg f66f7c8f5a remove override_dh_systemd_start 2 months ago
Max Erenberg 3e5b829085 check if mail_local_addresses exists in UWLDAP entry 2 months ago
Rio Liu 57ba72ef26 Add support for using number in member terms renwewal API (#77) 2 months ago
Max Erenberg 779e35a08e fix shadowExpire deserialization 3 months ago
Raymond Li 3cc9b011c3 Use HTTPS in sample 3 months ago
Max Erenberg 2739c45aff use LDAP instead of NSS for authz (#73) 3 months ago
Max Erenberg 651f4fb702 add more logging (#72) 3 months ago
Max Erenberg 953bee549e fix tests 3 months ago
Max Erenberg 0334e7e667 fix email formatting bug in ClubWebHostingService 3 months ago
Max Erenberg 8decd3bc30 packaging for bullseye 3 months ago
Max Erenberg 8ad8271db1 Fix some bugs in ClubWebHostingService 3 months ago
Raymond Li 4ebb9bb0a8
Release v1.0.22 4 months ago
  1. 7
      .drone/common.sh
  2. 4
      PACKAGING.md
  3. 15
      README.md
  4. 2
      VERSION.txt
  5. 7
      ceo/cli/members.py
  6. 28
      ceo/term_utils.py
  7. 6
      ceo/tui/controllers/AddUserController.py
  8. 2
      ceo/tui/controllers/RenewUserController.py
  9. 4
      ceo/tui/views/AddUserView.py
  10. 2
      ceo/utils.py
  11. 3
      ceo_common/interfaces/ICloudStackService.py
  12. 5
      ceo_common/interfaces/ILDAPService.py
  13. 14
      ceo_common/model/HTTPClient.py
  14. 37
      ceo_common/model/Term.py
  15. 60
      ceod/api/members.py
  16. 37
      ceod/api/utils.py
  17. 2
      ceod/model/CloudResourceManager.py
  18. 3
      ceod/model/CloudStackService.py
  19. 36
      ceod/model/ClubWebHostingService.py
  20. 4
      ceod/model/ContainerRegistryService.py
  21. 4
      ceod/model/KubernetesService.py
  22. 26
      ceod/model/LDAPService.py
  23. 5
      ceod/model/User.py
  24. 27
      debian/changelog
  25. 4
      debian/control
  26. 2
      debian/rules
  27. 10
      dev-requirements.txt
  28. 5
      docs/openapi.yaml
  29. 2842
      docs/redoc-static.html
  30. 10
      tests/ceo/cli/test_groups.py
  31. 19
      tests/ceo/cli/test_members.py
  32. 3
      tests/ceo/cli/test_webhosting.py
  33. 6
      tests/ceod/api/test_groups.py
  34. 68
      tests/ceod/api/test_members.py
  35. 4
      tests/ceod/api/test_webhosting.py
  36. 1
      tests/ceod/model/test_user.py
  37. 44
      tests/ceod/model/test_webhosting.py
  38. 5
      tests/ceod_test_local.ini
  39. 91
      tests/conftest.py
  40. 14
      tests/utils.py

@ -2,7 +2,12 @@
chmod 1777 /tmp
# don't resolve container names to *real* CSC machines
sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
sed -E 's/([[:alnum:]-]+\.)*uwaterloo\.ca//g' /etc/resolv.conf > /tmp/resolv.conf
# remove empty 'search' lines, if we created them
sed -E -i '/^search[[:space:]]*$/d' /tmp/resolv.conf
# also remove the 'rotate' option, since this can cause the Docker DNS server
# to be circumvented
sed -E -i '/^options.*\brotate/d' /tmp/resolv.conf
cp /tmp/resolv.conf /etc/resolv.conf
rm /tmp/resolv.conf

@ -9,6 +9,8 @@ Make sure your GPG key is in /srv/debian/gpg on potassium-benzoate. See
[here](https://wiki.csclub.uwaterloo.ca/Debian_Repository#Step_1:_Add_to_Uploaders)
for instructions.
Make sure you are in the `csc-mirror` group too.
## Creating the package
Use Docker/Podman to avoid screwing up your main system.
For example, to create a package for bullseye (replace `podman` with `docker` in all instances below if you're using Docker):
@ -58,7 +60,7 @@ podman cp pyceo-packaging:/home/max/repos/pyceo.tar.gz .
(Replace `/home/max/repos` by the directory in the container with the tarball.)
Now upload the tarball to a CSC machine, e.g.
```
scp pyceo.tar.gz mannitol:~/
scp pyceo.tar.gz mannitol:~
```
SSH into that machine and extract the tarball into a separate directory:
```

@ -16,7 +16,7 @@ Docker containers instead, which are much easier to work with than the VM.
First, make sure you create the virtualenv:
```sh
docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
docker run --rm -v "$PWD:$PWD:z" -w "$PWD" python:3.7-buster sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
```
Then bring up the containers:
```sh
@ -214,14 +214,23 @@ curl -V
```
Your should see 'SPNEGO' in the 'Features' section.
Here's an example of making a request to an endpoint which writes to LDAP:
Here's an example of making a request to add a user (in the Docker container):
```sh
# Get a Kerberos TGT first
# If you're root, switch to the ctdalek user first
su ctdalek
# Get a Kerberos TGT (password is krb5)
kinit
# Make the request
curl --negotiate -u : --service-name ceod --delegation always \
-d '{"uid":"test_1","cn":"Test One","given_name":"Test","sn":"One","program":"Math","terms":["s2021"]}' \
-X POST http://phosphoric-acid:9987/api/members
# To delete the user:
curl --negotiate -u : --service-name ceod --delegation always \
-X DELETE http://phosphoric-acid:9987/api/members/test_1
# In prod, use the following base URL instead:
# https://phosphoric-acid.csclub.uwaterloo.ca:9987
```
## Packaging

@ -1 +1 @@
1.0.21
1.0.24

@ -4,12 +4,13 @@ from typing import Dict
import click
from zope import component
from ..term_utils import get_terms_for_new_user, get_terms_for_renewal
from ..term_utils import get_terms_for_renewal_for_user
from ..utils import http_post, http_get, http_patch, http_delete, \
get_failed_operations, user_dict_lines, get_adduser_operations
from .utils import handle_stream_response, handle_sync_response, print_lines, \
check_if_in_development
from ceo_common.interfaces import IConfig
from ceo_common.model.Term import get_terms_for_new_user
from ceod.transactions.members import DeleteMemberTransaction
@ -48,7 +49,7 @@ def add(username, cn, given_name, sn, program, num_terms, clubrep, forwarding_ad
sn = result['sn']
if program is None and result.get('program'):
program = result['program']
if forwarding_address is None:
if forwarding_address is None and result.get('mail_local_addresses'):
forwarding_address = result['mail_local_addresses'][0]
if cn is None:
cn = click.prompt('Full name')
@ -155,7 +156,7 @@ def modify(username, login_shell, forwarding_addresses):
@click.option('--clubrep', is_flag=True, default=False,
help='Add non-member terms instead of member terms')
def renew(username, num_terms, clubrep):
terms = get_terms_for_renewal(username, num_terms, clubrep)
terms = get_terms_for_renewal_for_user(username, num_terms, clubrep)
if clubrep:
body = {'non_member_terms': terms}

@ -1,38 +1,22 @@
from typing import List
from .utils import http_get
from ceo_common.model import Term
from ceo_common.model.Term import get_terms_for_renewal
import ceo.cli.utils as cli_utils
import ceo.tui.utils as tui_utils
# Had to put these in a separate file to avoid a circular import.
def get_terms_for_new_user(num_terms: int) -> List[str]:
current_term = Term.current()
terms = [current_term + i for i in range(num_terms)]
return list(map(str, terms))
def get_terms_for_renewal(
def get_terms_for_renewal_for_user(
username: str, num_terms: int, clubrep: bool, tui_controller=None,
) -> List[str]:
resp = http_get('/api/members/' + username)
# FIXME: this is ugly, we shouldn't need a hacky if statement like this
if tui_controller is None:
result = cli_utils.handle_sync_response(resp)
else:
result = tui_utils.handle_sync_response(resp, tui_controller)
max_term = None
current_term = Term.current()
if clubrep and 'non_member_terms' in result:
max_term = max(Term(s) for s in result['non_member_terms'])
elif not clubrep and 'terms' in result:
max_term = max(Term(s) for s in result['terms'])
if max_term is not None and max_term >= current_term:
next_term = max_term + 1
if clubrep:
return get_terms_for_renewal(result.get('non_member_terms'), num_terms)
else:
next_term = Term.current()
terms = [next_term + i for i in range(num_terms)]
return list(map(str, terms))
return get_terms_for_renewal(result.get('terms'), num_terms)

@ -3,9 +3,9 @@ from threading import Thread
from ...utils import http_get
from .Controller import Controller
from .AddUserTransactionController import AddUserTransactionController
import ceo.term_utils as term_utils
from ceo.tui.models import TransactionModel
from ceo.tui.views import AddUserConfirmationView, TransactionView
from ceo_common.model.Term import get_terms_for_new_user
from ceod.transactions.members import AddMemberTransaction
@ -26,7 +26,7 @@ class AddUserController(Controller):
body['program'] = self.model.program
if self.model.forwarding_address:
body['forwarding_addresses'] = [self.model.forwarding_address]
new_terms = term_utils.get_terms_for_new_user(self.model.num_terms)
new_terms = get_terms_for_new_user(self.model.num_terms)
if self.model.membership_type == 'club_rep':
body['non_member_terms'] = new_terms
else:
@ -106,5 +106,5 @@ class AddUserController(Controller):
self.model.first_name = data.get('given_name', '')
self.model.last_name = data.get('sn', '')
self.model.program = data.get('program', '')
self.model.forwarding_address = data.get('mail_local_addresses', [''])[0]
self.model.forwarding_address = (data.get('mail_local_addresses') or [''])[0]
self.app.run_in_main_loop(self._on_lookup_user_success)

@ -28,7 +28,7 @@ class RenewUserController(SyncRequestController):
def _get_next_terms(self):
try:
self.model.new_terms = term_utils.get_terms_for_renewal(
self.model.new_terms = term_utils.get_terms_for_renewal_for_user(
self.model.username,
self.model.num_terms,
self.model.membership_type == 'club_rep',

@ -53,6 +53,10 @@ class AddUserView(ColumnView):
urwid.Text('Program:', align='right'),
self.program_edit
),
(
urwid.Text('Forwarding address:', align='right'),
self.forwarding_address_edit
),
(
urwid.Text('Number of terms:', align='right'),
self.num_terms_edit

@ -136,6 +136,8 @@ def user_dict_kv(d: Dict) -> List[Tuple[str]]:
pairs.append(('non-member terms', ','.join(_terms)))
if 'password' in d:
pairs.append(('password', d['password']))
if 'groups' in d:
pairs.append(('groups', ','.join(d['groups'])))
return pairs

@ -19,8 +19,7 @@ class ICloudStackService(Interface):
The dict is mapping of usernames to account IDs.
"""
def delete_account(account_id: str):
def delete_account(username: str, account_id: str):
"""
Delete the given CloudStack account.
Note that a CloudStack account ID must be given, not a username.
"""

@ -33,6 +33,11 @@ class ILDAPService(Interface):
A new UID and GID will be generated and returned in the new user.
"""
def get_groups_for_user(username: str) -> List[str]:
"""
Get a list of the groups to which the user belongs.
"""
def remove_user(user: IUser):
"""Remove this user from the database."""

@ -6,7 +6,7 @@ from requests_gssapi import HTTPSPNEGOAuth
from zope import component
from zope.interface import implementer
from ceo_common.interfaces import IConfig, IHTTPClient
from ceo_common.interfaces import IConfig, IHTTPClient, IKerberosService
@implementer(IHTTPClient)
@ -40,10 +40,18 @@ class HTTPClient:
'opportunistic_auth': True,
'target_name': gssapi.Name('ceod/' + host),
}
if flask.has_request_context() and 'client_token' in g:
if flask.has_request_context():
# This is reached when we are the server and the client has
# forwarded their credentials to us.
spnego_kwargs['creds'] = gssapi.Credentials(token=g.client_token)
token = None
if g.get('need_admin_creds', False):
# Some Kerberos bindings in some programming languages can't
# perform delegation, so use the admin creds here.
token = component.getUtility(IKerberosService).get_admin_creds_token()
elif 'client_token' in g:
token = g.client_token
if token is not None:
spnego_kwargs['creds'] = gssapi.Credentials(token=token)
elif delegate:
# This is reached when we are the client and we want to
# forward our credentials to the server.

@ -1,4 +1,5 @@
import datetime
from typing import List, Union
import ceo_common.utils as utils
@ -81,3 +82,39 @@ class Term:
month = self.seasons.index(c) * 4 + 1
day = 1
return datetime.datetime(year, month, day)
# Utility functions
def get_terms_for_new_user(num_terms: int) -> List[str]:
current_term = Term.current()
terms = [current_term + i for i in range(num_terms)]
return list(map(str, terms))
def get_terms_for_renewal(
existing_terms: Union[List[str], None],
num_terms: int,
) -> List[str]:
"""Calculates the terms for which a member or club rep should be renewed.
:param terms: The existing terms for the user being renewed. If the user
is being renewed as a regular member, these should be the
member terms. If they are being renewed as a club rep, these
should be the non-member terms.
This may be None if the user does not have any terms of the
appropriate type (an empty list is also acceptable).
:param num_terms: The number of terms for which the user is being renewed.
"""
max_term = None
current_term = Term.current()
if existing_terms:
max_term = max(map(Term, existing_terms))
if max_term is not None and max_term >= current_term:
next_term = max_term + 1
else:
next_term = current_term
terms = [next_term + i for i in range(num_terms)]
return list(map(str, terms))

@ -1,13 +1,14 @@
from flask import Blueprint, g, request
from flask import Blueprint, request
from flask.json import jsonify
from zope import component
from .utils import authz_restrict_to_staff, authz_restrict_to_syscom, \
user_is_in_group, requires_authentication_no_realm, \
user_is_in_group, requires_authentication_no_realm, requires_admin_creds, \
create_streaming_response, development_only, is_truthy
from ceo_common.errors import BadRequest, UserAlreadySubscribedError, UserNotSubscribedError
from ceo_common.interfaces import ILDAPService, IConfig, IMailService
from ceo_common.logger_factory import logger_factory
from ceo_common.model.Term import get_terms_for_new_user, get_terms_for_renewal
from ceod.transactions.members import (
AddMemberTransaction,
ModifyMemberTransaction,
@ -20,20 +21,29 @@ logger = logger_factory(__name__)
@bp.route('/', methods=['POST'], strict_slashes=False)
@requires_admin_creds
@authz_restrict_to_staff
def create_user():
# We need to use the admin creds here because office members may not
# directly create new LDAP records.
body = request.get_json(force=True)
terms = body.get('terms')
non_member_terms = body.get('non_member_terms')
if (terms and non_member_terms) or not (terms or non_member_terms):
raise BadRequest('Must specify either terms or non-member terms')
if type(terms) is int:
terms = get_terms_for_new_user(terms)
elif type(non_member_terms) is int:
non_member_terms = get_terms_for_new_user(non_member_terms)
for attr in ['uid', 'cn', 'given_name', 'sn']:
if not body.get(attr):
raise BadRequest(f"Attribute '{attr}' is missing or empty")
# We need to use the admin creds here because office members may not
# directly create new LDAP records.
g.need_admin_creds = True
if terms:
logger.info(f"Creating member {body['uid']} for terms {terms}")
else:
logger.info(f"Creating club rep {body['uid']} for non-member terms {non_member_terms}")
txn = AddMemberTransaction(
uid=body['uid'],
@ -61,7 +71,9 @@ def get_user(auth_user: str, username: str):
get_forwarding_addresses = True
ldap_srv = component.getUtility(ILDAPService)
user = ldap_srv.get_user(username)
return user.to_dict(get_forwarding_addresses)
user_dict = user.to_dict(get_forwarding_addresses)
user_dict['groups'] = ldap_srv.get_groups_for_user(username)
return user_dict
@bp.route('/<username>', methods=['PATCH'])
@ -81,40 +93,48 @@ def patch_user(auth_user: str, username: str):
@bp.route('/<username>/renew', methods=['POST'])
@requires_admin_creds
@authz_restrict_to_staff
def renew_user(username: str):
# We need to use the admin creds here because office members should
# not be able to directly modify the shadowExpire field; this could
# prevent syscom members from logging into the machines.
body = request.get_json(force=True)
terms = body.get('terms')
non_member_terms = body.get('non_member_terms')
if (terms and non_member_terms) or not (terms or non_member_terms):
raise BadRequest('Must specify either terms or non-member terms')
# We need to use the admin creds here because office members should
# not be able to directly modify the shadowExpire field; this could
# prevent syscom members from logging into the machines.
g.need_admin_creds = True
ldap_srv = component.getUtility(ILDAPService)
cfg = component.getUtility(IConfig)
user = ldap_srv.get_user(username)
member_list = cfg.get('mailman3_new_member_list')
if type(terms) is int:
terms = get_terms_for_renewal(user.terms, terms)
elif type(non_member_terms) is int:
non_member_terms = get_terms_for_renewal(user.non_member_terms, non_member_terms)
def unexpire(user):
if user.shadowExpire:
user.set_expired(False)
try:
user.subscribe_to_mailing_list(member_list)
logger.debug(f'Unsubscribed {user.uid} from {member_list}')
except UserAlreadySubscribedError:
pass
logger.debug(f'{user.uid} is already unsubscribed from {member_list}')
if body.get('terms'):
user.add_terms(body['terms'])
if terms:
logger.info(f"Renewing member {username} for terms {terms}")
user.add_terms(terms)
unexpire(user)
return {'terms_added': body['terms']}
elif body.get('non_member_terms'):
user.add_non_member_terms(body['non_member_terms'])
return {'terms_added': terms}
elif non_member_terms:
logger.info(f"Renewing club rep {username} for non-member terms {non_member_terms}")
user.add_non_member_terms(non_member_terms)
unexpire(user)
return {'non_member_terms_added': body['non_member_terms']}
return {'non_member_terms_added': non_member_terms}
else:
raise BadRequest('Must specify either terms or non-member terms')
@ -149,11 +169,13 @@ def expire_users():
if not dry_run:
for member in members:
logger.info(f'Expiring {member.uid}')
member.set_expired(True)
try:
member.unsubscribe_from_mailing_list(member_list)
logger.debug(f'Unsubscribed {member.uid} from {member_list}')
except UserNotSubscribedError:
pass
logger.debug(f'{member.uid} is already unsubscribed from {member_list}')
return jsonify([member.uid for member in members])

@ -1,12 +1,9 @@
import functools
import grp
import json
import os
import pwd
import traceback
from typing import Callable, List
from flask import current_app, stream_with_context
from flask import current_app, g, stream_with_context
from zope import component
from .spnego import requires_authentication
@ -40,9 +37,24 @@ def requires_authentication_no_realm(f: Callable) -> Callable:
return wrapper
def user_is_in_group(user: str, group: str) -> bool:
"""Returns True if `user` is in `group`, False otherwise."""
return user in grp.getgrnam(group).gr_mem
def requires_admin_creds(f: Callable) -> Callable:
"""
Forces the next LDAP connection to use the admin Kerberos credentials.
This must be used BEFORE any of the authz decorators, since those
may require an LDAP connection, which will get cached for later use.
"""
@functools.wraps(f)
def wrapper(*args, **kwargs):
g.need_admin_creds = True
return f(*args, **kwargs)
return wrapper
def user_is_in_group(username: str, group_name: str) -> bool:
"""Returns True if `username` is in `group_name`, False otherwise."""
ldap_srv = component.getUtility(ILDAPService)
group = ldap_srv.get_group(group_name)
return username in group.members
def authz_restrict_to_groups(f: Callable, allowed_groups: List[str]) -> Callable:
@ -51,8 +63,6 @@ def authz_restrict_to_groups(f: Callable, allowed_groups: List[str]) -> Callable
specified groups.
"""
allowed_group_ids = [grp.getgrnam(g).gr_gid for g in allowed_groups]
@requires_authentication_no_realm
@functools.wraps(f)
def wrapper(_username: str, *args, **kwargs):
@ -62,15 +72,14 @@ def authz_restrict_to_groups(f: Callable, allowed_groups: List[str]) -> Callable
if username.startswith('ceod/'):
# ceod services are always allowed to make internal calls
return f(*args, **kwargs)
for gid in os.getgrouplist(username, pwd.getpwnam(username).pw_gid):
if gid in allowed_group_ids:
ldap_srv = component.getUtility(ILDAPService)
for group_name in ldap_srv.get_groups_for_user(username):
if group_name in allowed_groups:
return f(*args, **kwargs)
logger.debug(
f"User '{username}' denied since they are not in one of {allowed_groups}"
)
return {
'error': f'You must be in one of {allowed_groups}'
}, 403
return {'error': f'You must be in one of {allowed_groups}'}, 403
return wrapper

@ -97,7 +97,7 @@ class CloudResourceManager:
resources = accounts[username]['resources']
if 'cloudstack' in resources:
account_id = accounts[username]['cloudstack_account_id']
cloudstack_srv.delete_account(account_id)
cloudstack_srv.delete_account(username, account_id)
if 'vhost' in resources:
vhost_mgr.delete_all_vhosts_for_user(username)
if 'k8s' in resources:

@ -76,7 +76,8 @@ class CloudStackService:
for account in d['account']
}
def delete_account(self, account_id: str):
def delete_account(self, username: str, account_id: str):
logger.info(f'Deleting CloudStack account for {username}')
url = self._create_url({
'command': 'deleteAccount',
'id': account_id,

@ -84,16 +84,19 @@ class ClubWebHostingService:
logger.debug('Reloading Apache')
self._run(['systemctl', 'reload', 'apache2'])
# This requires the APACHE_CONFIG_CRON environment variable to be
# set to 1 (e.g. in a systemd drop-in)
# See /etc/apache2/.git/hooks/pre-commit on caffeine
def _git_commit(self):
if not os.path.isdir(os.path.join(self.apache_dir, '.git')):
logger.debug('No git folder found in Apache directory')
return
logger.debug('Committing changes to git repository')
self._run(['git', 'add', APACHE_DISABLED_CLUBS_FILE], cwd=self.apache_dir)
self._run(['git', 'commit', '-m', '[ceo] disable club websites'], cwd=self.apache_dir)
self._run(
['git', 'add', APACHE_DISABLED_CLUBS_FILE],
cwd=self.apache_dir)
# See /etc/apache2/.git/hooks/pre-commit on caffeine
self._run(
['git', 'commit', '-m', '[ceo] disable club websites'],
cwd=self.apache_dir,
env={**os.environ, 'APACHE_CONFIG_CRON': '1'})
def commit(self):
if not self.made_at_least_one_change:
@ -112,12 +115,13 @@ class ClubWebHostingService:
directive_paths = self.aug.match(f'/files/etc/apache2/sites-available/{filename}/VirtualHost/directive')
for directive_path in directive_paths:
directive = self.aug.get(directive_path)
directive_value = self.aug.get(directive_path + '/arg')
if directive == 'DocumentRoot':
directive_value = self.aug.get(directive_path + '/arg')
match = APACHE_USERDIR_RE.match(directive_value)
if match is not None:
club_name = match.group('club_name')
elif directive == 'ServerAdmin':
directive_value = self.aug.get(directive_path + '/arg')
club_email = directive_value
if club_name is not None:
self.clubs[club_name]['email'] = club_email
@ -157,12 +161,20 @@ class ClubWebHostingService:
def _site_uses_php(self, club_name: str) -> bool:
www = f'{self.clubs_home}/{club_name}/www'
if os.path.isdir(www):
if not os.path.isdir(www):
return False
try:
# We're just going to look one level deep; that should be good enough.
for filename in os.listdir(www):
filepath = os.path.join(www, filename)
if os.path.isfile(filepath) and filename.endswith('.php'):
return True
filenames = os.listdir(www)
except os.error:
# If we're unable to read the directory (e.g. permissions error),
# then this means that the Apache user (www-data) can't read it either.
# So we can just return False here.
return False
for filename in filenames:
filepath = os.path.join(www, filename)
if os.path.isfile(filepath) and filename.endswith('.php'):
return True
return False
# This method needs to be called from within a transaction (uses self.clubs)
@ -224,7 +236,7 @@ class ClubWebHostingService:
# STEP 2: send emails to clubs whose websites were disabled
clubs_who_were_not_notified = set()
for club_name in clubs_to_disable:
address = clubs_info['email']
address = clubs_info[club_name]['email']
if address is None:
clubs_who_were_not_notified.add(club_name)
continue

@ -7,6 +7,9 @@ from zope.interface import implementer
from ceo_common.errors import UserNotFoundError
from ceo_common.interfaces import IContainerRegistryService, IConfig
from ceo_common.logger_factory import logger_factory
logger = logger_factory(__name__)
@implementer(IContainerRegistryService)
@ -70,6 +73,7 @@ class ContainerRegistryService:
resp.raise_for_status()
def delete_project_for_user(self, username: str):
logger.info(f'Deleting Harbor project for {username}')
# Delete all of the repositories inside the project first
resp = self._http_get(f'/projects/{username}/repositories')
if resp.status_code == 403:

@ -11,6 +11,9 @@ from zope import component
from zope.interface import implementer
from ceo_common.interfaces import IConfig, IKubernetesService
from ceo_common.logger_factory import logger_factory
logger = logger_factory(__name__)
@implementer(IKubernetesService)
@ -100,6 +103,7 @@ class KubernetesService:
return body
def delete_account(self, username: str):
logger.info(f'Deleting Kubernetes namespace for {username}')
namespace = self._get_namespace(username)
# don't check exit code because namespace might not exist
self._run(['kubectl', 'delete', 'namespace', namespace], check=False)

@ -98,6 +98,13 @@ class LDAPService:
entry = self._get_readable_entry_for_group(conn, cn)
return Group.deserialize_from_ldap(entry)
def get_groups_for_user(self, username: str) -> List[str]:
conn = self._get_ldap_conn()
conn.search(self.ldap_groups_base,
f'(uniqueMember={self.uid_to_dn(username)})',
attributes=['cn'])
return sorted([entry.cn.value for entry in conn.entries])
def get_display_info_for_users(self, usernames: List[str]) -> List[Dict[str, str]]:
if not usernames:
return []
@ -105,14 +112,14 @@ class LDAPService:
filter = '(|' + ''.join([f'(uid={uid})' for uid in usernames]) + ')'
attributes = ['uid', 'cn', 'program']
conn.search(self.ldap_users_base, filter, attributes=attributes)
return [
return sorted([
{
'uid': entry.uid.value,
'cn': entry.cn.value,
'program': entry.program.value or 'Unknown',
}
for entry in conn.entries
]
], key=lambda member: member['uid'])
def get_users_with_positions(self) -> List[IUser]:
conn = self._get_ldap_conn()
@ -309,12 +316,12 @@ class LDAPService:
self,
dry_run: bool = False,
members: Union[List[str], None] = None,
uwldap_batch_size: int = 10,
uwldap_batch_size: int = 100,
):
if members:
filter = '(|' + ''.join([f'(uid={uid})' for uid in members]) + ')'
else:
filter = '(objectClass=*)'
filter = '(objectClass=member)'
conn = self._get_ldap_conn()
conn.search(
self.ldap_users_base, filter, attributes=['uid', 'program'])
@ -329,12 +336,17 @@ class LDAPService:
batch_uids = uids[i:i + uwldap_batch_size]
batch_uw_programs = uwldap_srv.get_programs_for_users(batch_uids)
uw_programs.extend(batch_uw_programs)
# uw_programs[i] will be None if the 'ou' attribute was not
# present in UWLDAP, or if no UWLDAP entry was found at all
for i, uw_program in enumerate(uw_programs):
if uw_program in (None, 'expired', 'orphaned'):
# If the UWLDAP record is orphaned, nonexistent, or missing
# data, assume that the member graduated
uw_programs[i] = 'Alumni'
users_to_change = [
(uids[i], csc_programs[i], uw_programs[i])
for i in range(len(uids))
if csc_programs[i] != uw_programs[i] and (
uw_programs[i] not in (None, 'expired', 'orphaned')
)
if csc_programs[i] != uw_programs[i]
]
if dry_run:
return users_to_change

@ -86,7 +86,6 @@ class User:
'is_club': self.is_club(),
'is_club_rep': self.is_club_rep,
'program': self.program or 'Unknown',
'shadowExpire': self.shadowExpire,
}
if self.sn and self.given_name:
data['sn'] = self.sn
@ -103,6 +102,8 @@ class User:
data['mail_local_addresses'] = self.mail_local_addresses
if get_forwarding_addresses:
data['forwarding_addresses'] = self.get_forwarding_addresses()
if self.shadowExpire:
data['shadow_expire'] = self.shadowExpire
return data
def __repr__(self) -> str:
@ -169,7 +170,7 @@ class User:
is_club_rep=attrs.get('isClubRep', [False])[0],
is_club=('club' in attrs['objectClass']),
is_member_or_club_rep=('member' in attrs['objectClass']),
shadowExpire=attrs.get('shadowExpire'),
shadowExpire=attrs.get('shadowExpire', [None])[0],
ldap3_entry=entry,
)

27
debian/changelog vendored

@ -1,3 +1,30 @@
ceo (1.0.24-bullseye1) bullseye; urgency=high
* Add support for using number in member terms renwewal API
* Sort group member listing by WatIAM ID
* Add more logging for Cloudstack
* Use LDAP instead of NSS
* Fix shadowExpire deserialization
* Fix email formatting bug in ClubWebHostingService
* Check if mail_local_addresses exists in UWLDAP entry
* Remove override_dh_systemd_start
-- Max Erenberg <merenber@csclub.uwaterloo.ca> Sun, 23 Oct 2022 21:41:00 -0400
ceo (1.0.23-bullseye1) bullseye; urgency=high
* Fix some bugs in ClubWebHostingService.
-- Max Erenberg <merenber@csclub.uwaterloo.ca> Mon, 05 Sep 2022 03:57:47 +0000
ceo (1.0.22-bullseye1) bullseye; urgency=medium
* Implement renewal reminders.
* Allow addmember and removemember to accept multiple usernames.
* Disable inactive club sites
-- Raymond Li <raymo@csclub.uwaterloo.ca> Sat, 06 Aug 2022 01:43:08 +0000
ceo (1.0.21-bullseye1) bullseye; urgency=high
* Fix bug in ContainerRegistryService.

4
debian/control vendored

@ -5,7 +5,9 @@ Priority: optional
Standards-Version: 4.3.0
Vcs-Git: https://git.csclub.uwaterloo.ca/public/pyceo.git
Vcs-Browser: https://git.csclub.uwaterloo.ca/public/pyceo
Uploaders: Max Erenberg <merenber@csclub.uwaterloo.ca>
Uploaders: Max Erenberg <merenber@csclub.uwaterloo.ca>,
Raymond Li <raymo@csclub.uwaterloo.ca>,
Edwin <e42zhang@csclub.uwaterloo.ca>
Build-Depends: debhelper (>= 12.1.1),
python3-dev (>= 3.7),
python3-venv (>= 3.7),

2
debian/rules vendored

@ -7,5 +7,3 @@ override_dh_strip:
override_dh_shlibdeps:
override_dh_systemd_start:
dh_systemd_start --no-start ceod.service

@ -1,6 +1,6 @@
flake8==3.9.2
setuptools==40.8.0
wheel==0.36.2
pytest==6.2.4
flake8==5.0.4
setuptools==65.4.1
wheel==0.37.1
pytest==7.1.3
aiosmtpd==1.4.2
aiohttp==3.7.4.post0
aiohttp==3.8.3

@ -942,6 +942,11 @@ components:
$ref: "#/components/schemas/NonMemberTerms"
forwarding_addresses:
$ref: "#/components/schemas/ForwardingAddresses"
groups:
type: array
description: Groups for which this user is a member of
items:
$ref: "#/components/schemas/GroupCN"
UWLDAPUser:
type: object
properties:

File diff suppressed because one or more lines are too long

@ -117,8 +117,7 @@ def test_groups_multiple_members(cli_setup, new_user_gen):
def test_groups_with_auxiliary_groups_and_mailing_lists(cli_setup, ldap_user):
runner = CliRunner()
# make sure auxiliary groups + mailing lists exist in ceod_test_local.ini
create_group('syscom', 'Systems Committee')
create_group('office', 'Office')
create_group('adm', 'Administrators')
create_group('staff', 'Staff')
runner = CliRunner()
@ -131,7 +130,7 @@ def test_groups_with_auxiliary_groups_and_mailing_lists(cli_setup, ldap_user):
"Add user to auxiliary groups... Done\n"
"Subscribe user to auxiliary mailing lists... Done\n"
"Transaction successfully completed.\n"
f"Added {ldap_user.uid} to syscom, office, staff\n"
f"Added {ldap_user.uid} to syscom, adm, staff\n"
f"Subscribed {ldap_user.uid} to syscom@csclub.internal, syscom-alerts@csclub.internal\n"
)
assert result.exit_code == 0
@ -147,7 +146,7 @@ def test_groups_with_auxiliary_groups_and_mailing_lists(cli_setup, ldap_user):
"Remove user from auxiliary groups... Done\n"
"Unsubscribe user from auxiliary mailing lists... Done\n"
"Transaction successfully completed.\n"
f"Removed {ldap_user.uid} from syscom, office, staff\n"
f"Removed {ldap_user.uid} from syscom, adm, staff\n"
f"Unsubscribed {ldap_user.uid} from syscom@csclub.internal, syscom-alerts@csclub.internal\n"
)
assert result.exit_code == 0
@ -167,6 +166,5 @@ def test_groups_with_auxiliary_groups_and_mailing_lists(cli_setup, ldap_user):
assert result.exit_code == 0
assert 'Unsubscribed' not in result.output
delete_group('syscom')
delete_group('office')
delete_group('adm')
delete_group('staff')

@ -4,6 +4,7 @@ import shutil
from datetime import datetime, timedelta
from click.testing import CliRunner
import ldap3
from ceo.cli import cli
from ceo_common.model import Term
@ -25,8 +26,9 @@ def test_members_get(cli_setup, ldap_user):
f"home directory: {ldap_user.home_directory}\n"
f"is a club: {ldap_user.is_club()}\n"
f"is a club rep: {ldap_user.is_club_rep}\n"
"forwarding addresses: \n"
f"forwarding addresses: \n"
f"member terms: {','.join(ldap_user.terms)}\n"
f"groups: \n"
)
assert result.exit_code == 0
assert result.output == expected
@ -146,7 +148,7 @@ def test_members_pwreset(cli_setup, ldap_user, krb_user):
assert expected_pat.match(result.output) is not None
def test_members_expire(cli_setup, app_process, ldap_user, syscom_group):
def test_members_expire(cli_setup, app_process, ldap_user):
runner = CliRunner()
try:
@ -172,12 +174,17 @@ def test_members_expire(cli_setup, app_process, ldap_user, syscom_group):
restore_datetime_in_app_process(app_process)
def test_members_remindexpire(cli_setup, app_process, ldap_user):
def test_members_remindexpire(cfg, cli_setup, app_process, ldap_conn, ldap_user):
runner = CliRunner()
term = Term(ldap_user.terms[0])
test_date = (term + 1).to_datetime()
# Add a term to ctdalek so that he doesn't show up in the results
base_dn = cfg.get('ldap_users_base')
ldap_conn.modify(
f'uid=ctdalek,{base_dn}',
{'term': [(ldap3.MODIFY_ADD, [str(term + 1)])]})
try:
test_date = (term + 1).to_datetime()
set_datetime_in_app_process(app_process, test_date)
result = runner.invoke(cli, ['members', 'remindexpire', '--dry-run'])
assert result.exit_code == 0
@ -200,3 +207,7 @@ def test_members_remindexpire(cli_setup, app_process, ldap_user):
assert result.output == "No members are pending expiration.\n"
finally:
restore_datetime_in_app_process(app_process)
ldap_conn.modify(
f'uid=ctdalek,{base_dn}',
{'term': [(ldap3.MODIFY_DELETE, [str(term + 1)])]})

@ -5,6 +5,7 @@ from ceo_common.model import Term
from tests.utils import (
create_php_file_for_club,
reset_disable_club_conf,
create_website_config_for_club,
set_datetime_in_app_process,
restore_datetime_in_app_process,
)
@ -15,10 +16,12 @@ def test_disable_club_sites(
new_club_gen, new_user_gen, g_admin_ctx, ldap_srv_session,
):
runner = CliRunner()
sites_available_dir = webhosting_srv.sites_available_dir
term = Term.current()
clubs_home = cfg.get('clubs_home')
with new_club_gen() as group, new_user_gen() as user:
create_php_file_for_club(clubs_home, group.cn)
create_website_config_for_club(sites_available_dir, group.cn)
user.add_non_member_terms([str(Term.current())])
group.add_member(user.uid)

@ -125,7 +125,8 @@ def test_api_group_auxiliary(cfg, client, ldap_user, g_admin_ctx):
group_names = ['syscom'] + aux_groups
groups = []
with g_admin_ctx():
for group_name in group_names:
# the syscom group should already exist, since we need it for auth
for group_name in aux_groups:
group = Group(
cn=group_name,
gid_number=min_uid,
@ -161,4 +162,5 @@ def test_api_group_auxiliary(cfg, client, ldap_user, g_admin_ctx):
with g_admin_ctx():
for group in groups:
group.remove_from_ldap()
if group.cn != 'syscom':
group.remove_from_ldap()

@ -72,7 +72,6 @@ def test_api_create_user(cfg, create_user_resp, mock_mail_server):
"mail_local_addresses": ["test1@csclub.internal"],
"forwarding_addresses": ['test1@uwaterloo.internal'],
"password": "krb5",
"shadowExpire": None,
}},
]
assert data == expected
@ -84,6 +83,25 @@ def test_api_create_user(cfg, create_user_resp, mock_mail_server):
mock_mail_server.messages.clear()
def test_api_create_user_with_num_terms(client):
status, data = client.post('/api/members', json={
'uid': 'test2',
'cn': 'Test Two',
'given_name': 'Test',
'sn': 'Two',
'program': 'Math',
'terms': 2,
'forwarding_addresses': ['test2@uwaterloo.internal'],
})
assert status == 200
assert data[-1]['status'] == 'completed'
current_term = Term.current()
assert data[-1]['result']['terms'] == [str(current_term), str(current_term + 1)]
status, data = client.delete('/api/members/test2')
assert status == 200
assert data[-1]['status'] == 'completed'
def test_api_next_uid(cfg, client, create_user_result):
min_uid = cfg.get('members_min_id')
_, data = client.post('/api/members', json={
@ -109,6 +127,7 @@ def test_api_get_user(cfg, client, create_user_result):
del old_data['password']
status, data = client.get(f'/api/members/{uid}')
del data['groups']
assert status == 200
assert data == old_data
@ -203,6 +222,20 @@ def test_api_renew_user(cfg, client, create_user_result, ldap_conn):
ldap_conn.modify(dn, changes)
def test_api_renew_user_with_num_terms(client, ldap_user):
uid = ldap_user.uid
status, data = client.post(f'/api/members/{uid}/renew', json={'terms': 2})
assert status == 200
_, data = client.get(f'/api/members/{uid}')
current_term = Term.current()
assert data['terms'] == [str(current_term), str(current_term + 1), str(current_term + 2)]
status, data = client.post(f'/api/members/{uid}/renew', json={'non_member_terms': 2})
assert status == 200
_, data = client.get(f'/api/members/{uid}')
assert data['non_member_terms'] == [str(current_term), str(current_term + 1)]
def test_api_reset_password(client, create_user_result):
uid = create_user_result['uid']