merge upstream

pull/10/head
Andrew Wang 1 year ago
commit 5893e561cd
  1. 14
      .drone.yml
  2. 20
      .drone/auth1-setup.sh
  3. 15
      .drone/phosphoric-acid-setup.sh
  4. 1
      .drone/slapd.conf
  5. 27
      README.md
  6. 4
      ceo/__main__.py
  7. 1
      ceo/cli/__init__.py
  8. 48
      ceo/cli/entrypoint.py
  9. 148
      ceo/cli/groups.py
  10. 218
      ceo/cli/members.py
  11. 35
      ceo/cli/updateprograms.py
  12. 121
      ceo/cli/utils.py
  13. 24
      ceo/krb_check.py
  14. 27
      ceo/operation_strings.py
  15. 59
      ceo/utils.py
  16. 8
      ceo_common/errors.py
  17. 5
      ceo_common/interfaces/IGroup.py
  18. 19
      ceo_common/interfaces/IHTTPClient.py
  19. 3
      ceo_common/interfaces/ILDAPService.py
  20. 16
      ceo_common/interfaces/IMailService.py
  21. 7
      ceo_common/interfaces/IUser.py
  22. 4
      ceo_common/model/Config.py
  23. 57
      ceo_common/model/HTTPClient.py
  24. 9
      ceo_common/model/RemoteMailmanService.py
  25. 67
      ceo_common/model/Term.py
  26. 1
      ceo_common/model/__init__.py
  27. 11
      ceod/api/app_factory.py
  28. 3
      ceod/api/error_handlers.py
  29. 10
      ceod/api/members.py
  30. 45
      ceod/api/positions.py
  31. 55
      ceod/api/spnego.py
  32. 16
      ceod/api/utils.py
  33. 8
      ceod/model/Group.py
  34. 20
      ceod/model/KerberosService.py
  35. 11
      ceod/model/LDAPService.py
  36. 35
      ceod/model/MailService.py
  37. 14
      ceod/model/User.py
  38. 10
      ceod/model/templates/announce_new_user.j2
  39. 14
      ceod/model/templates/welcome_message.j2
  40. 4
      ceod/transactions/AbstractTransaction.py
  41. 21
      ceod/transactions/members/AddMemberTransaction.py
  42. 109
      ceod/transactions/members/UpdateMemberPositionsTransaction.py
  43. 1
      ceod/transactions/members/__init__.py
  44. 2
      requirements.txt
  45. 4
      tests/MockMailmanServer.py
  46. 0
      tests/ceo/__init__.py
  47. 0
      tests/ceo/cli/__init__.py
  48. 154
      tests/ceo/cli/test_groups.py
  49. 135
      tests/ceo/cli/test_members.py
  50. 43
      tests/ceo/cli/test_updatemembers.py
  51. 48
      tests/ceo_common/model/test_remote_mailman.py
  52. 9
      tests/ceo_dev.ini
  53. 6
      tests/ceod/api/test_members.py
  54. 93
      tests/ceod/api/test_positions.py
  55. 4
      tests/ceod/model/test_group.py
  56. 2
      tests/ceod/model/test_mail.py
  57. 14
      tests/ceod/model/test_user.py
  58. 6
      tests/ceod_dev.ini
  59. 8
      tests/ceod_test_local.ini
  60. 131
      tests/conftest.py
  61. 18
      tests/conftest_ceo.py
  62. 89
      tests/conftest_ceod_api.py
  63. 34
      tests/utils.py

@ -1,19 +1,20 @@
kind: pipeline
type: docker
name: phosphoric-acid
name: default
steps:
- name: run tests
# use the step name to mock out the gethostname() call in our tests
- name: phosphoric-acid
image: python:3.7-buster
# unfortunately we have to do everything in one step because there's no
# way to share system packages between steps
commands:
# install dependencies
# install dependencies
- apt update && apt install -y libkrb5-dev libsasl2-dev python3-dev
- python3 -m venv venv
- . venv/bin/activate
- pip install -r requirements.txt
- pip install -r dev-requirements.txt
- pip install -r requirements.txt
- cd ceo_common/krb5 && python krb5_build.py && cd ../..
# lint
@ -29,3 +30,8 @@ services:
commands:
- .drone/auth1-setup.sh
- sleep infinity
trigger:
branch:
- master
- v1

@ -2,17 +2,18 @@
set -ex
# sanity check
test $(hostname) = auth1
# don't resolve container names to *real* CSC machines
sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
cat /tmp/resolv.conf > /etc/resolv.conf
rm /tmp/resolv.conf
get_ip_addr() {
getent hosts $1 | cut -d' ' -f1
}
add_fqdn_to_hosts() {
hostname=$1
ip_addr=$(getent hosts $hostname | cut -d' ' -f1)
ip_addr=$1
hostname=$2
sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
cat /tmp/hosts > /etc/hosts
rm /tmp/hosts
@ -20,7 +21,14 @@ add_fqdn_to_hosts() {
}
# set FQDN in /etc/hosts
add_fqdn_to_hosts auth1
add_fqdn_to_hosts $(get_ip_addr $(hostname)) auth1
# I'm not sure why, but we also need to remove the hosts entry for the
# container's real hostname, otherwise slapd only looks for the principal
# ldap/<container hostname> (this is with the sasl-host option)
sed -E "/\\b$(hostname)\\b/d" /etc/hosts > /tmp/hosts
cat /tmp/hosts > /etc/hosts
rm /tmp/hosts
export DEBIAN_FRONTEND=noninteractive
apt update

@ -2,17 +2,18 @@
set -ex
# sanity check
test $(hostname) = phosphoric-acid
# don't resolve container names to *real* CSC machines
sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
cat /tmp/resolv.conf > /etc/resolv.conf
rm /tmp/resolv.conf
get_ip_addr() {
getent hosts $1 | cut -d' ' -f1
}
add_fqdn_to_hosts() {
hostname=$1
ip_addr=$(getent hosts $hostname | cut -d' ' -f1)
ip_addr=$1
hostname=$2
sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
cat /tmp/hosts > /etc/hosts
rm /tmp/hosts
@ -20,8 +21,8 @@ add_fqdn_to_hosts() {
}
# set FQDN in /etc/hosts
add_fqdn_to_hosts phosphoric-acid
add_fqdn_to_hosts auth1
add_fqdn_to_hosts $(get_ip_addr $(hostname)) phosphoric-acid
add_fqdn_to_hosts $(get_ip_addr auth1) auth1
export DEBIAN_FRONTEND=noninteractive
apt update

@ -30,6 +30,7 @@ localssf 128
# map kerberos users to ldap users
sasl-realm CSCLUB.INTERNAL
sasl-host auth1.csclub.internal
authz-regexp "uid=([^/=]*),cn=CSCLUB.INTERNAL,cn=GSSAPI,cn=auth"
"uid=$1,ou=people,dc=csclub,dc=internal"
authz-regexp "uid=ceod/admin,cn=CSCLUB.INTERNAL,cn=GSSAPI,cn=auth"

@ -65,6 +65,22 @@ host all all 0.0.0.0/0 reject
systemctl restart postgresql
```
#### Mailman
You should create the following mailing lists from the mail container:
```sh
/opt/mailman3/bin/mailman create syscom@csclub.internal
/opt/mailman3/bin/mailman create syscom-alerts@csclub.internal
/opt/mailman3/bin/mailman create exec@csclub.internal
/opt/mailman3/bin/mailman create ceo@csclub.internal
```
See https://git.uwaterloo.ca/csc/syscom-dev-environment/-/tree/master/mail
for instructions on how to access the Mailman UI from your browser.
If you want to actually see the archived messages, you'll
need to tweak the settings for each list from the UI so that non-member
messages get accepted (by default they get held).
#### Dependencies
Next, install and activate a virtualenv:
```sh
@ -138,14 +154,3 @@ curl --negotiate -u : --service-name ceod \
-d '{"uid":"test_1","cn":"Test One","program":"Math","terms":["s2021"]}' \
-X POST http://phosphoric-acid:9987/api/members
```
## Miscellaneous
### Mailman
You may wish to add more mailing lists to Mailman; by default, only the
csc-general list exists (from the dev environment playbooks). Just
attach to the mail container and run the following:
```sh
/opt/mailman3/bin/mailman create new_list_name@csclub.internal
```
See https://git.uwaterloo.ca/csc/syscom-dev-environment/-/tree/master/mail
for instructions on how to access the Mailman UI from your browser.

@ -0,0 +1,4 @@
from .cli import cli
if __name__ == '__main__':
cli(obj={})

@ -0,0 +1 @@
from .entrypoint import cli

@ -0,0 +1,48 @@
import importlib.resources
import os
import socket
import click
from zope import component
from ..krb_check import krb_check
from .members import members
from .groups import groups
from .updateprograms import updateprograms
from ceo_common.interfaces import IConfig, IHTTPClient
from ceo_common.model import Config, HTTPClient
@click.group()
@click.pass_context
def cli(ctx):
# ensure ctx exists and is a dict
ctx.ensure_object(dict)
princ = krb_check()
user = princ[:princ.index('@')]
ctx.obj['user'] = user
if os.environ.get('PYTEST') != '1':
register_services()
cli.add_command(members)
cli.add_command(groups)
cli.add_command(updateprograms)
def register_services():
# Config
# This is a hack to determine if we're in the dev env or not
if socket.getfqdn().endswith('.csclub.internal'):
with importlib.resources.path('tests', 'ceo_dev.ini') as p:
config_file = p.__fspath__()
else:
config_file = os.environ.get('CEO_CONFIG', '/etc/csc/ceo.ini')
cfg = Config(config_file)
component.provideUtility(cfg, IConfig)
# HTTPService
http_client = HTTPClient()
component.provideUtility(http_client, IHTTPClient)

@ -0,0 +1,148 @@
from typing import Dict
import click
from zope import component
from ..utils import http_post, http_get, http_delete
from .utils import handle_stream_response, handle_sync_response, print_colon_kv, \
check_if_in_development
from ceo_common.interfaces import IConfig
from ceod.transactions.groups import (
AddGroupTransaction,
AddMemberToGroupTransaction,
RemoveMemberFromGroupTransaction,
DeleteGroupTransaction,
)
@click.group(short_help='Perform operations on CSC groups/clubs')
def groups():
pass
@groups.command(short_help='Add a new group')
@click.argument('group_name')
@click.option('-d', '--description', help='Group description', prompt=True)
def add(group_name, description):
click.echo('The following group will be created:')
lines = [
('cn', group_name),
('description', description),
]
print_colon_kv(lines)
click.confirm('Do you want to continue?', abort=True)
body = {
'cn': group_name,
'description': description,
}
operations = AddGroupTransaction.operations
resp = http_post('/api/groups', json=body)
data = handle_stream_response(resp, operations)
result = data[-1]['result']
print_group_lines(result)
def print_group_lines(result: Dict):
"""Pretty-print a group JSON response."""
lines = [
('cn', result['cn']),
('description', result.get('description', 'Unknown')),
('gid_number', str(result['gid_number'])),
]
for i, member in enumerate(result['members']):
if i == 0:
prefix = 'members'
else:
prefix = ''
lines.append((prefix, member['cn'] + ' (' + member['uid'] + ')'))
print_colon_kv(lines)
@groups.command(short_help='Get info about a group')
@click.argument('group_name')
def get(group_name):
resp = http_get('/api/groups/' + group_name)
result = handle_sync_response(resp)
print_group_lines(result)
@groups.command(short_help='Add a member to a group')
@click.argument('group_name')
@click.argument('username')
@click.option('--no-subscribe', is_flag=True, default=False,
help='Do not subscribe the member to any auxiliary mailing lists.')
def addmember(group_name, username, no_subscribe):
click.confirm(f'Are you sure you want to add {username} to {group_name}?',
abort=True)
base_domain = component.getUtility(IConfig).get('base_domain')
url = f'/api/groups/{group_name}/members/{username}'
operations = AddMemberToGroupTransaction.operations
if no_subscribe:
url += '?subscribe_to_lists=false'
operations.remove('subscribe_user_to_auxiliary_mailing_lists')
resp = http_post(url)
data = handle_stream_response(resp, operations)
result = data[-1]['result']
lines = []
for i, group in enumerate(result['added_to_groups']):
if i == 0:
prefix = 'Added to groups'
else:
prefix = ''
lines.append((prefix, group))
for i, mailing_list in enumerate(result.get('subscribed_to_lists', [])):
if i == 0:
prefix = 'Subscribed to lists'
else:
prefix = ''
if '@' not in mailing_list:
mailing_list += '@' + base_domain
lines.append((prefix, mailing_list))
print_colon_kv(lines)
@groups.command(short_help='Remove a member from a group')
@click.argument('group_name')
@click.argument('username')
@click.option('--no-unsubscribe', is_flag=True, default=False,
help='Do not unsubscribe the member from any auxiliary mailing lists.')
def removemember(group_name, username, no_unsubscribe):
click.confirm(f'Are you sure you want to remove {username} from {group_name}?',
abort=True)
base_domain = component.getUtility(IConfig).get('base_domain')
url = f'/api/groups/{group_name}/members/{username}'
operations = RemoveMemberFromGroupTransaction.operations
if no_unsubscribe:
url += '?unsubscribe_from_lists=false'
operations.remove('unsubscribe_user_from_auxiliary_mailing_lists')
resp = http_delete(url)
data = handle_stream_response(resp, operations)
result = data[-1]['result']
lines = []
for i, group in enumerate(result['removed_from_groups']):
if i == 0:
prefix = 'Removed from groups'
else:
prefix = ''
lines.append((prefix, group))
for i, mailing_list in enumerate(result.get('unsubscribed_from_lists', [])):
if i == 0:
prefix = 'Unsubscribed from lists'
else:
prefix = ''
if '@' not in mailing_list:
mailing_list += '@' + base_domain
lines.append((prefix, mailing_list))
print_colon_kv(lines)
@groups.command(short_help='Delete a group')
@click.argument('group_name')
def delete(group_name):
check_if_in_development()
click.confirm(f"Are you sure you want to delete {group_name}?", abort=True)
resp = http_delete(f'/api/groups/{group_name}')
handle_stream_response(resp, DeleteGroupTransaction.operations)

@ -0,0 +1,218 @@
import sys
from typing import Dict
import click
from zope import component
from ..utils import http_post, http_get, http_patch, http_delete, get_failed_operations
from .utils import handle_stream_response, handle_sync_response, print_colon_kv, \
check_if_in_development
from ceo_common.interfaces import IConfig
from ceo_common.model import Term
from ceod.transactions.members import (
AddMemberTransaction,
DeleteMemberTransaction,
)
@click.group(short_help='Perform operations on CSC members and club reps')
def members():
pass
@members.command(short_help='Add a new member or club rep')
@click.argument('username')
@click.option('--cn', help='Full name', prompt='Full name')
@click.option('--program', required=False, help='Academic program')
@click.option('--terms', 'num_terms', type=click.IntRange(1, 100),
help='Number of terms to add', prompt='Number of terms')
@click.option('--clubrep', is_flag=True, default=False,
help='Add non-member terms instead of member terms')
@click.option('--forwarding-address', required=False,
help=('Forwarding address to set in ~/.forward. '
'Default is UW address. '
'Set to the empty string to disable forwarding.'))
def add(username, cn, program, num_terms, clubrep, forwarding_address):
cfg = component.getUtility(IConfig)
uw_domain = cfg.get('uw_domain')
current_term = Term.current()
terms = [current_term + i for i in range(num_terms)]
terms = list(map(str, terms))
if forwarding_address is None:
forwarding_address = username + '@' + uw_domain
click.echo("The following user will be created:")
lines = [
('uid', username),
('cn', cn),
]
if program is not None:
lines.append(('program', program))
if clubrep:
lines.append(('non-member terms', ','.join(terms)))
else:
lines.append(('member terms', ','.join(terms)))
if forwarding_address != '':
lines.append(('forwarding address', forwarding_address))
print_colon_kv(lines)
click.confirm('Do you want to continue?', abort=True)
body = {
'uid': username,
'cn': cn,
}
if program is not None:
body['program'] = program
if clubrep:
body['non_member_terms'] = terms
else:
body['terms'] = terms
if forwarding_address != '':
body['forwarding_addresses'] = [forwarding_address]
operations = AddMemberTransaction.operations
if forwarding_address == '':
# don't bother displaying this because it won't be run
operations.remove('set_forwarding_addresses')
resp = http_post('/api/members', json=body)
data = handle_stream_response(resp, operations)
result = data[-1]['result']
print_user_lines(result)
failed_operations = get_failed_operations(data)
if 'send_welcome_message' in failed_operations:
click.echo(click.style(
'Warning: welcome message was not sent. You now need to manually '
'send the user their password.', fg='yellow'))
def print_user_lines(result: Dict):
"""Pretty-print a user JSON response."""
lines = [
('uid', result['uid']),
('cn', result['cn']),
('program', result.get('program', 'Unknown')),
('UID number', result['uid_number']),
('GID number', result['gid_number']),
('login shell', result['login_shell']),
('home directory', result['home_directory']),
('is a club', result['is_club']),
]
if 'forwarding_addresses' in result:
if len(result['forwarding_addresses']) != 0:
lines.append(('forwarding addresses', result['forwarding_addresses'][0]))
for address in result['forwarding_addresses'][1:]:
lines.append(('', address))
if 'terms' in result:
lines.append(('terms', ','.join(result['terms'])))
if 'non_member_terms' in result:
lines.append(('non-member terms', ','.join(result['non_member_terms'])))
if 'password' in result:
lines.append(('password', result['password']))
print_colon_kv(lines)
@members.command(short_help='Get info about a user')
@click.argument('username')
def get(username):
resp = http_get('/api/members/' + username)
result = handle_sync_response(resp)
print_user_lines(result)
@members.command(short_help="Replace a user's login shell or forwarding addresses")
@click.argument('username')
@click.option('--login-shell', required=False, help='Login shell')
@click.option('--forwarding-addresses', required=False,
help=(
'Comma-separated list of forwarding addresses. '
'Set to the empty string to disable forwarding.'
))
def modify(username, login_shell, forwarding_addresses):
if login_shell is None and forwarding_addresses is None:
click.echo('Nothing to do.')
sys.exit()
operations = []
body = {}
if login_shell is not None:
body['login_shell'] = login_shell
operations.append('replace_login_shell')
click.echo('Login shell will be set to: ' + login_shell)
if forwarding_addresses is not None:
if forwarding_addresses == '':
forwarding_addresses = []
else:
forwarding_addresses = forwarding_addresses.split(',')
body['forwarding_addresses'] = forwarding_addresses
operations.append('replace_forwarding_addresses')
prefix = '~/.forward will be set to: '
if len(forwarding_addresses) > 0:
click.echo(prefix + forwarding_addresses[0])
for address in forwarding_addresses[1:]:
click.echo((' ' * len(prefix)) + address)
else:
click.echo(prefix)
click.confirm('Do you want to continue?', abort=True)
resp = http_patch('/api/members/' + username, json=body)
handle_stream_response(resp, operations)
@members.command(short_help="Renew a member or club rep's membership")
@click.argument('username')
@click.option('--terms', 'num_terms', type=click.IntRange(1, 100),
help='Number of terms to add', prompt='Number of terms')
@click.option('--clubrep', is_flag=True, default=False,
help='Add non-member terms instead of member terms')
def renew(username, num_terms, clubrep):
resp = http_get('/api/members/' + username)
result = handle_sync_response(resp)
max_term = None
current_term = Term.current()
if clubrep and 'non_member_terms' in result:
max_term = max(Term(s) for s in result['non_member_terms'])
elif not clubrep and 'terms' in result:
max_term = max(Term(s) for s in result['terms'])
if max_term is not None and max_term >= current_term:
next_term = max_term + 1
else:
next_term = Term.current()
terms = [next_term + i for i in range(num_terms)]
terms = list(map(str, terms))
if clubrep:
body = {'non_member_terms': terms}
click.echo('The following non-member terms will be added: ' + ','.join(terms))
else:
body = {'terms': terms}
click.echo('The following member terms will be added: ' + ','.join(terms))
click.confirm('Do you want to continue?', abort=True)
resp = http_post(f'/api/members/{username}/renew', json=body)
handle_sync_response(resp)
click.echo('Done.')
@members.command(short_help="Reset a user's password")
@click.argument('username')
def pwreset(username):
click.confirm(f"Are you sure you want to reset {username}'s password?", abort=True)
resp = http_post(f'/api/members/{username}/pwreset')
result = handle_sync_response(resp)
click.echo('New password: ' + result['password'])
@members.command(short_help="Delete a user")
@click.argument('username')
def delete(username):
check_if_in_development()
click.confirm(f"Are you sure you want to delete {username}?", abort=True)
resp = http_delete(f'/api/members/{username}')
handle_stream_response(resp, DeleteMemberTransaction.operations)

@ -0,0 +1,35 @@
import click
from ..utils import http_post
from .utils import handle_sync_response, print_colon_kv
@click.command(short_help="Sync the 'program' attribute with UWLDAP")
@click.option('--dry-run', is_flag=True, default=False)
@click.option('--members', required=False)
def updateprograms(dry_run, members):
body = {}
if dry_run:
body['dry_run'] = True
if members is not None:
body['members'] = ','.split(members)
if not dry_run:
click.confirm('Are you sure that you want to sync programs with UWLDAP?', abort=True)
resp = http_post('/api/uwldap/updateprograms', json=body)
result = handle_sync_response(resp)
if len(result) == 0:
click.echo('All programs are up-to-date.')
return
if dry_run:
click.echo('Members whose program would be changed:')
else:
click.echo('Members whose program was changed:')
lines = []
for uid, csc_program, uw_program in result:
csc_program = csc_program or 'Unknown'
csc_program = click.style(csc_program, fg='yellow')
uw_program = click.style(uw_program, fg='green')
lines.append((uid, csc_program + ' -> ' + uw_program))
print_colon_kv(lines)

@ -0,0 +1,121 @@
import json
import socket
import sys
from typing import List, Tuple, Dict
import click
import requests
from ..operation_strings import descriptions as op_desc
class Abort(click.ClickException):
"""Abort silently."""
def __init__(self, exit_code=1):
super().__init__('')
self.exit_code = exit_code
def show(self):
pass
def print_colon_kv(pairs: List[Tuple[str, str]]):
"""
Pretty-print a list of key-value pairs such that the key and value
columns align.
Example:
key1: value1
key1000: value2
"""
maxlen = max(len(key) for key, val in pairs)
for key, val in pairs:
if key != '':
click.echo(key + ': ', nl=False)
else:
# assume this is a continuation from the previous line
click.echo(' ', nl=False)
extra_space = ' ' * (maxlen - len(key))
click.echo(extra_space, nl=False)
click.echo(val)
def handle_stream_response(resp: requests.Response, operations: List[str]) -> List[Dict]:
"""
Print output to the console while operations are being streamed
from the server over HTTP.
Returns the parsed JSON data streamed from the server.
"""
if resp.status_code != 200:
click.echo('An error occurred:')
click.echo(resp.text.rstrip())
raise Abort()
click.echo(op_desc[operations[0]] + '... ', nl=False)
idx = 0
data = []
for line in resp.iter_lines(decode_unicode=True, chunk_size=8):
d = json.loads(line)
data.append(d)
if d['status'] == 'aborted':
click.echo(click.style('ABORTED', fg='red'))
click.echo('The transaction was rolled back.')
click.echo('The error was: ' + d['error'])
click.echo('Please check the ceod logs.')
sys.exit(1)
elif d['status'] == 'completed':
if idx < len(operations):
click.echo('Skipped')
click.echo('Transaction successfully completed.')
return data
operation = d['operation']
oper_failed = False
err_msg = None
prefix = 'failed_to_'
if operation.startswith(prefix):
operation = operation[len(prefix):]
oper_failed = True
# sometimes the operation looks like
# "failed_to_do_something: error message"
if ':' in operation:
operation, err_msg = operation.split(': ', 1)
while idx < len(operations) and operations[idx] != operation:
click.echo('Skipped')
idx += 1
if idx == len(operations):
break
click.echo(op_desc[operations[idx]] + '... ', nl=False)
if idx == len(operations):
click.echo('Unrecognized operation: ' + operation)
continue
if oper_failed:
click.echo(click.style('Failed', fg='red'))
if err_msg is not None:
click.echo(' Error message: ' + err_msg)
else:
click.echo(click.style('Done', fg='green'))
idx += 1
if idx < len(operations):
click.echo(op_desc[operations[idx]] + '... ', nl=False)
raise Exception('server response ended abruptly')
def handle_sync_response(resp: requests.Response):
"""
Exit the program if the request was not successful.
Returns the parsed JSON response.
"""
if resp.status_code != 200:
click.echo('An error occurred:')
click.echo(resp.text.rstrip())
raise Abort()
return resp.json()
def check_if_in_development() -> bool:
"""Aborts if we are not currently in the dev environment."""
if not socket.getfqdn().endswith('.csclub.internal'):
click.echo('This command may only be called during development.')
raise Abort()

@ -0,0 +1,24 @@
import subprocess
import gssapi
def krb_check():
"""
Spawns a `kinit` process if no credentials are available or the
credentials have expired.
Returns the principal string 'user@REALM'.
"""
for _ in range(2):
try:
creds = gssapi.Credentials(usage='initiate')
result = creds.inquire()
return str(result.name)
except (gssapi.raw.misc.GSSError, gssapi.raw.exceptions.ExpiredCredentialsError):
kinit()
raise Exception('could not acquire GSSAPI credentials')
def kinit():
subprocess.run(['kinit'], check=True)

@ -0,0 +1,27 @@
# These descriptions are printed to the console while a transaction
# is performed, in real time.
descriptions = {
'add_user_to_ldap': 'Add user to LDAP',
'add_group_to_ldap': 'Add group to LDAP',
'add_user_to_kerberos': 'Add user to Kerberos',
'create_home_dir': 'Create home directory',
'set_forwarding_addresses': 'Set forwarding addresses',
'send_welcome_message': 'Send welcome message',
'subscribe_to_mailing_list': 'Subscribe to mailing list',
'announce_new_user': 'Announce new user to mailing list',
'replace_login_shell': 'Replace login shell',
'replace_forwarding_addresses': 'Replace forwarding addresses',
'remove_user_from_ldap': 'Remove user from LDAP',
'remove_group_from_ldap': 'Remove group from LDAP',
'remove_user_from_kerberos': 'Remove user from Kerberos',
'delete_home_dir': 'Delete home directory',
'unsubscribe_from_mailing_list': 'Unsubscribe from mailing list',
'add_sudo_role': 'Add sudo role to LDAP',
'add_user_to_group': 'Add user to group',
'add_user_to_auxiliary_groups': 'Add user to auxiliary groups',
'subscribe_user_to_auxiliary_mailing_lists': 'Subscribe user to auxiliary mailing lists',
'remove_user_from_group': 'Remove user from group',
'remove_user_from_auxiliary_groups': 'Remove user from auxiliary groups',
'unsubscribe_user_from_auxiliary_mailing_lists': 'Unsubscribe user from auxiliary mailing lists',
'remove_sudo_role': 'Remove sudo role from LDAP',
}

@ -0,0 +1,59 @@
from typing import List, Dict
import requests
from zope import component
from ceo_common.interfaces import IHTTPClient, IConfig
def http_request(method: str, path: str, **kwargs) -> requests.Response:
client = component.getUtility(IHTTPClient)
cfg = component.getUtility(IConfig)
if path.startswith('/api/db'):
host = cfg.get('ceod_db_host')
need_cred = False
else:
host = cfg.get('ceod_admin_host')
# The forwarded TGT is only needed for endpoints which write to LDAP
need_cred = method != 'GET'
return client.request(
host, path, method, principal=None, need_cred=need_cred,
stream=True, **kwargs)
def http_get(path: str, **kwargs) -> requests.Response:
return http_request('GET', path, **kwargs)
def http_post(path: str, **kwargs) -> requests.Response:
return http_request('POST', path, **kwargs)
def http_patch(path: str, **kwargs) -> requests.Response:
return http_request('PATCH', path, **kwargs)
def http_delete(path: str, **kwargs) -> requests.Response:
return http_request('DELETE', path, **kwargs)
def get_failed_operations(data: List[Dict]) -> List[str]:
"""
Get a list of the failed operations using the JSON objects
streamed from the server.
"""
prefix = 'failed_to_'
failed = []
for d in data:
if 'operation' not in d:
continue
operation = d['operation']
if not operation.startswith(prefix):
continue
operation = operation[len(prefix):]
if ':' in operation:
# sometimes the operation looks like
# "failed_to_do_something: error message"
operation = operation[:operation.index(':')]
failed.append(operation)
return failed

@ -1,11 +1,11 @@
class UserNotFoundError(Exception):
def __init__(self):
super().__init__('user not found')
def __init__(self, username):
super().__init__(f"user '{username}' not found")
class GroupNotFoundError(Exception):
def __init__(self):
super().__init__('group not found')
def __init__(self, group_name):
super().__init__(f"group '{group_name}' not found")
class BadRequest(Exception):

@ -1,3 +1,5 @@
from typing import List
from zope.interface import Interface, Attribute
@ -21,5 +23,8 @@ class IGroup(Interface):
def remove_member(username: str):
"""Remove the member from this group in LDAP."""
def set_members(usernames: List[str]):
"""Set all of the members of this group in LDAP."""
def to_dict():
"""Serialize this group as JSON."""

@ -1,14 +1,27 @@
from typing import Union
from zope.interface import Interface
class IHTTPClient(Interface):
"""A helper class for HTTP requests to ceod."""
def get(host: str, api_path: str, **kwargs):
def request(host: str, api_path: str, method: str, principal: str,
need_cred: bool, **kwargs):
"""Make an HTTP request."""
def get(host: str, api_path: str, principal: Union[str, None] = None,
need_cred: bool = True, **kwargs):
"""Make a GET request."""
def post(host: str, api_path: str, **kwargs):
def post(host: str, api_path: str, principal: Union[str, None] = None,
need_cred: bool = True, **kwargs):
"""Make a POST request."""
def delete(host: str, api_path: str, **kwargs):
def patch(host: str, api_path: str, principal: Union[str, None] = None,
need_cred: bool = True, **kwargs):
"""Make a PATCH request."""
def delete(host: str, api_path: str, principal: Union[str, None] = None,
need_cred: bool = True, **kwargs):
"""Make a DELETE request."""

@ -24,6 +24,9 @@ class ILDAPService(Interface):
Useful for displaying a list of users in a compact way.
"""
def get_users_with_positions(self) -> List[IUser]:
"""Retrieve users who have a non-empty position attribute."""
def add_user(user: IUser):
"""
Add the user to the database.

@ -1,4 +1,4 @@
from typing import Dict
from typing import Dict, List
from zope.interface import Interface
@ -11,5 +11,15 @@ class IMailService(Interface):
def send(_from: str, to: str, headers: Dict[str, str], content: str):
"""Send a message with the given headers and content."""
def send_welcome_message_to(user: IUser):
"""Send a welcome message to the new member."""
def send_welcome_message_to(user: IUser, password: str):
"""
Send a welcome message to the new member, including their temporary
password.
"""
def announce_new_user(user: IUser, operations: List[str]):
"""
Announce to the ceo mailing list that the new user was created.
`operations` is a list of the operations which were performed
during the transaction.
"""

@ -53,11 +53,8 @@ class IUser(Interface):
def add_non_member_terms(terms: List[str]):
"""Add non-member terms for this user."""
def add_position(position: str):
"""Add a position to this user."""
def remove_position(position: str):
"""Remove a position from this user."""
def set_positions(self, positions: List[str]):
"""Set the positions for this user."""
def change_password(password: str):
"""Replace this user's password."""

@ -27,6 +27,6 @@ class Config:
return True
if val.lower() in ['false', 'no']:
return False
if section.startswith('auxiliary '):
return val.split(',')
if section.startswith('auxiliary ') or section == 'positions':
return [item.strip() for item in val.split(',')]
return val

@ -1,10 +1,13 @@
from flask import g
from base64 import b64encode
from typing import Union
import gssapi
import requests
from requests_gssapi import HTTPSPNEGOAuth
from zope import component
from zope.interface import implementer
from ceo_common.krb5.utils import get_fwd_tgt
from ceo_common.interfaces import IConfig, IHTTPClient
@ -20,34 +23,48 @@ class HTTPClient:
self.ceod_port = cfg.get('ceod_port')
self.base_domain = cfg.get('base_domain')
self.krb_realm = cfg.get('ldap_sasl_realm')
def request(self, host: str, api_path: str, method: str, principal: str,
need_cred: bool, **kwargs):
# always use the FQDN
if '.' not in host:
host = host + '.' + self.base_domain
def request(self, host: str, api_path: str, method='GET', **kwargs):
principal = g.sasl_user
if '@' not in principal:
principal = principal + '@' + self.krb_realm
gssapi_name = gssapi.Name(principal)
creds = gssapi.Credentials(name=gssapi_name, usage='initiate')
# SPNEGO
if principal is not None:
gssapi_name = gssapi.Name(principal)
creds = gssapi.Credentials(name=gssapi_name, usage='initiate')
else:
creds = None
auth = HTTPSPNEGOAuth(
opportunistic_auth=True,
target_name='ceod',
target_name=gssapi.Name('ceod/' + host),
creds=creds,
)
# always use the FQDN, for HTTPS purposes
if '.' not in host:
host = host + '.' + self.base_domain
# Forwarded TGT (X-KRB5-CRED)
headers = {}
if need_cred:
b = get_fwd_tgt('ceod/' + host)
headers['X-KRB5-CRED'] = b64encode(b).decode()
return requests.request(
method,
f'{self.scheme}://{host}:{self.ceod_port}{api_path}',
auth=auth,
**kwargs,
auth=auth, headers=headers, **kwargs,
)
def get(self, host: str, api_path: str, **kwargs):
return self.request(host, api_path, 'GET', **kwargs)
def get(self, host: str, api_path: str, principal: Union[str, None] = None,
need_cred: bool = False, **kwargs):
return self.request(host, api_path, 'GET', principal, need_cred, **kwargs)
def post(self, host: str, api_path: str, principal: Union[str, None] = None,
need_cred: bool = False, **kwargs):
return self.request(host, api_path, 'POST', principal, need_cred, **kwargs)
def post(self, host: str, api_path: str, **kwargs):
return self.request(host, api_path, 'POST', **kwargs)
def patch(self, host: str, api_path: str, principal: Union[str, None] = None,
need_cred: bool = False, **kwargs):
return self.request(host, api_path, 'PATCH', principal, need_cred, **kwargs)
def delete(self, host: str, api_path: str, **kwargs):
return self.request(host, api_path, 'DELETE', **kwargs)
def delete(self, host: str, api_path: str, principal: Union[str, None] = None,
need_cred: bool = False, **kwargs):
return self.request(host, api_path, 'DELETE', principal, need_cred, **kwargs)

@ -1,3 +1,4 @@
from flask import g
from zope import component
from zope.interface import implementer
@ -14,7 +15,9 @@ class RemoteMailmanService:
self.http_client = component.getUtility(IHTTPClient)
def subscribe(self, address: str, mailing_list: str):
resp = self.http_client.post(self.mailman_host, f'/api/mailman/{mailing_list}/{address}')
resp = self.http_client.post(
self.mailman_host, f'/api/mailman/{mailing_list}/{address}',
principal=g.sasl_user)
if not resp.ok:
if resp.status_code == 409:
raise UserAlreadySubscribedError()
@ -23,7 +26,9 @@ class RemoteMailmanService:
raise Exception(resp.json())
def unsubscribe(self, address: str, mailing_list: str):
resp = self.http_client.delete(self.mailman_host, f'/api/mailman/{mailing_list}/{address}')
resp = self.http_client.delete(
self.mailman_host, f'/api/mailman/{mailing_list}/{address}',
principal=g.sasl_user)
if not resp.ok:
if resp.status_code == 404:
raise UserNotSubscribedError()

@ -0,0 +1,67 @@
import datetime
class Term:
"""A representation of a term in the CSC LDAP, e.g. 's2021'."""
seasons = ['w', 's', 'f']
def __init__(self, s_term: str):
assert len(s_term) == 5 and s_term[0] in self.seasons and \
s_term[1:].isdigit()
self.s_term = s_term
def __repr__(self):
return self.s_term
@staticmethod
def current():
"""Get a Term object for the current date."""
dt = datetime.datetime.now()
c = 'w'
if 5 <= dt.month <= 8:
c = 's'
elif 9 <= dt.month:
c = 'f'
s_term = c + str(dt.year)
return Term(s_term)
def __add__(self, other):
assert type(other) is int and other >= 0
c = self.s_term[0]
season_idx = self.seasons.index(c)
year = int(self.s_term[1:])
year += other // 3
season_idx += other % 3
if season_idx >= 3:
year += 1
season_idx -= 3
s_term = self.seasons[season_idx] + str(year)
return Term(s_term)
def __eq__(self, other):
return isinstance(other, Term) and self.s_term == other.s_term
def __lt__(self, other):
if not isinstance(other, Term):
return NotImplemented
c1, c2 = self.s_term[0], other.s_term[0]
year1, year2 = int(self.s_term[1:]), int(other.s_term[1:])
return year1 < year2 or (
year1 == year2 and self.seasons.index(c1) < self.seasons.index(c2)
)
def __gt__(self, other):
if not isinstance(other, Term):
return NotImplemented
c1, c2 = self.s_term[0], other.s_term[0]
year1, year2 = int(self.s_term[1:]), int(other.s_term[1:])
return year1 > year2 or (
year1 == year2 and self.seasons.index(c1) > self.seasons.index(c2)
)
def __ge__(self, other):
return self > other or self == other
def __le__(self, other):
return self < other or self == other

@ -1,3 +1,4 @@
from .Config import Config
from .HTTPClient import HTTPClient
from .RemoteMailmanService import RemoteMailmanService
from .Term import Term

@ -3,7 +3,6 @@ import os
import socket
from flask import Flask
from flask_kerberos import init_kerberos
from zope import component
from .error_handlers import register_error_handlers
@ -11,6 +10,7 @@ from .krb5_cred_handlers import before_request, teardown_request
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService
from ceo_common.model import Config, HTTPClient, RemoteMailmanService
from ceod.api.spnego import init_spnego
from ceod.model import KerberosService, LDAPService, FileService, \
MailmanService, MailService, UWLDAPService
from ceod.db import MySQLService, PostgreSQLService
@ -24,9 +24,7 @@ def create_app(flask_config={}):
register_services(app)
cfg = component.getUtility(IConfig)
fqdn = socket.getfqdn()
os.environ['KRB5_KTNAME'] = '/etc/krb5.keytab'
init_kerberos(app, service='ceod', hostname=fqdn)
init_spnego('ceod')
hostname = socket.gethostname()
# Only ceod_admin_host should serve the /api/members endpoints because
@ -43,6 +41,9 @@ def create_app(flask_config={}):
from ceod.api import groups
app.register_blueprint(groups.bp, url_prefix='/api/groups')
from ceod.api import positions
app.register_blueprint(positions.bp, url_prefix='/api/positions')
from ceod.api import uwldap
app.register_blueprint(uwldap.bp, url_prefix='/api/uwldap')
@ -69,8 +70,6 @@ def register_services(app):
component.provideUtility(cfg, IConfig)
# KerberosService
if 'KRB5_KTNAME' not in os.environ:
os.environ['KRB5_KTNAME'] = '/etc/krb5.keytab'
hostname = socket.gethostname()
fqdn = socket.getfqdn()
# Only ceod_admin_host has the ceod/admin key in its keytab

@ -1,6 +1,7 @@
import traceback
from flask.app import Flask
import ldap3
from werkzeug.exceptions import HTTPException
from ceo_common.errors import UserNotFoundError, GroupNotFoundError
@ -21,6 +22,8 @@ def generic_error_handler(err: Exception):
status_code = err.code
elif isinstance(err, UserNotFoundError) or isinstance(err, GroupNotFoundError):