Compare commits

..

31 Commits

Author SHA1 Message Date
Max Erenberg 754731ba5f upgrade Debian dependencies 3 days ago
Max Erenberg b33339817f fix logging messages for renewing a member 3 days ago
Max Erenberg 6dccd8b659 release 1.0.25 3 days ago
Max Erenberg 3f58d1aff5 print warning message when cloud account is created 3 days ago
Justin Chung 5e8f1b5ba5 Implement TUI support for multiple users in each position (#80) 2 weeks ago
Max Erenberg f84965c8e1 reload all NGINX servers after adding a vhost (#90) 2 weeks ago
Max Erenberg 4394c4e277 use bullseye for base container (#91) 2 weeks ago
Jonathan Leung b507c56136 Show groups in member for API, CLI and TUI (#82) 2 months ago
Max Erenberg c0c9736593 Use the admin creds in the HTTPClient when necessary (#85) 3 months ago
Max Erenberg 1e452d10ce Assume program is Alumni if UWLDAP is missing data (#84) 3 months ago
Max Erenberg 6a1fa81b82 merenber signs the packages 4 months ago
Max Erenberg 6df1f4d459 Revert "Simplify packaging" 4 months ago
Edwin 2cf9e25b59 More fixes 4 months ago
Edwin 9ff3d850c9 Release 1.0.24 4 months ago
Edwin b4a1373559 Simplify packaging 4 months ago
Raymond Li dceb5d6572
Revert "#63: Add positions to CEO (#79)" 4 months ago
Jonathan Leung c30ca54752 Sort group member listing by WatIAM ID (#78) 4 months ago
Nathan Chung 3b7c89c925 #63: Add positions to CEO (#79) 4 months ago
Max Erenberg a2324090f3 update README instructions for curl 4 months ago
Max Erenberg f66f7c8f5a remove override_dh_systemd_start 4 months ago
Max Erenberg 3e5b829085 check if mail_local_addresses exists in UWLDAP entry 4 months ago
Rio Liu 57ba72ef26 Add support for using number in member terms renwewal API (#77) 4 months ago
Max Erenberg 779e35a08e fix shadowExpire deserialization 5 months ago
Raymond Li 3cc9b011c3 Use HTTPS in sample 5 months ago
Max Erenberg 2739c45aff use LDAP instead of NSS for authz (#73) 5 months ago
Max Erenberg 651f4fb702 add more logging (#72) 5 months ago
Max Erenberg 953bee549e fix tests 5 months ago
Max Erenberg 0334e7e667 fix email formatting bug in ClubWebHostingService 5 months ago
Max Erenberg 8decd3bc30 packaging for bullseye 5 months ago
Max Erenberg 8ad8271db1 Fix some bugs in ClubWebHostingService 5 months ago
Raymond Li 4ebb9bb0a8
Release v1.0.22 6 months ago
  1. 6
      .drone.yml
  2. 1
      .drone/auth1-setup.sh
  3. 8
      .drone/coffee-setup.sh
  4. 8
      .drone/common.sh
  5. 4
      PACKAGING.md
  6. 15
      README.md
  7. 2
      VERSION.txt
  8. 5
      ceo/cli/cloud.py
  9. 7
      ceo/cli/members.py
  10. 15
      ceo/cli/positions.py
  11. 28
      ceo/term_utils.py
  12. 6
      ceo/tui/controllers/AddUserController.py
  13. 4
      ceo/tui/controllers/GetPositionsController.py
  14. 2
      ceo/tui/controllers/RenewUserController.py
  15. 6
      ceo/tui/controllers/SetPositionsController.py
  16. 4
      ceo/tui/views/AddUserView.py
  17. 5
      ceo/utils.py
  18. 3
      ceo_common/interfaces/ICloudStackService.py
  19. 5
      ceo_common/interfaces/ILDAPService.py
  20. 14
      ceo_common/model/HTTPClient.py
  21. 37
      ceo_common/model/Term.py
  22. 60
      ceod/api/members.py
  23. 43
      ceod/api/positions.py
  24. 37
      ceod/api/utils.py
  25. 2
      ceod/model/CloudResourceManager.py
  26. 3
      ceod/model/CloudStackService.py
  27. 36
      ceod/model/ClubWebHostingService.py
  28. 4
      ceod/model/ContainerRegistryService.py
  29. 4
      ceod/model/KubernetesService.py
  30. 26
      ceod/model/LDAPService.py
  31. 5
      ceod/model/User.py
  32. 11
      ceod/model/VHostManager.py
  33. 20
      ceod/transactions/members/UpdateMemberPositionsTransaction.py
  34. 35
      debian/changelog
  35. 2
      debian/compat
  36. 32
      debian/control
  37. 7
      debian/rules
  38. 10
      dev-requirements.txt
  39. 6
      docker-compose.yml
  40. 62
      docs/openapi.yaml
  41. 2847
      docs/redoc-static.html
  42. 1
      etc/ceod.ini
  43. 2
      requirements.txt
  44. 5
      tests/ceo/cli/test_cloud.py
  45. 14
      tests/ceo/cli/test_db_mysql.py
  46. 10
      tests/ceo/cli/test_groups.py
  47. 19
      tests/ceo/cli/test_members.py
  48. 80
      tests/ceo/cli/test_positions.py
  49. 3
      tests/ceo/cli/test_webhosting.py
  50. 6
      tests/ceod/api/test_groups.py
  51. 68
      tests/ceod/api/test_members.py
  52. 53
      tests/ceod/api/test_positions.py
  53. 4
      tests/ceod/api/test_webhosting.py
  54. 1
      tests/ceod/model/test_user.py
  55. 44
      tests/ceod/model/test_webhosting.py
  56. 1
      tests/ceod_dev.ini
  57. 6
      tests/ceod_test_local.ini
  58. 91
      tests/conftest.py
  59. 14
      tests/utils.py

@ -5,7 +5,7 @@ name: default
steps:
# use the step name to mock out the gethostname() call in our tests
- name: phosphoric-acid
image: python:3.7-buster
image: python:3.9-bullseye
# unfortunately we have to do everything in one step because there's no
# way to share system packages between steps
commands:
@ -25,12 +25,12 @@ steps:
services:
- name: auth1
image: debian:buster
image: debian:bullseye
commands:
- .drone/auth1-setup.sh
- sleep infinity
- name: coffee
image: debian:buster
image: debian:bullseye
commands:
- .drone/coffee-setup.sh
- sleep infinity

@ -28,7 +28,6 @@ killall slapd || true
service nslcd stop || true
rm -rf /etc/ldap/slapd.d
rm /var/lib/ldap/*
cp /usr/share/slapd/DB_CONFIG /var/lib/ldap/DB_CONFIG
cp .drone/slapd.conf /etc/ldap/slapd.conf
cp .drone/ldap.conf /etc/ldap/ldap.conf
cp /usr/share/doc/sudo-ldap/schema.OpenLDAP /etc/ldap/schema/sudo.schema

@ -11,9 +11,9 @@ add_fqdn_to_hosts $(get_ip_addr auth1) auth1
apt install --no-install-recommends -y default-mysql-server postgresql
# MYSQL
service mysql stop
sed -E -i 's/^(bind-address[[:space:]]+= 127.0.0.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf
service mysql start
service mariadb stop
sed -E -i 's/^(bind-address[[:space:]]+= 127\.0\.0\.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf
service mariadb start
cat <<EOF | mysql
CREATE USER IF NOT EXISTS 'mysql' IDENTIFIED BY 'mysql';
GRANT ALL PRIVILEGES ON *.* TO 'mysql' WITH GRANT OPTION;
@ -21,7 +21,7 @@ EOF
# POSTGRESQL
service postgresql stop
POSTGRES_DIR=/etc/postgresql/11/main
POSTGRES_DIR=/etc/postgresql/*/main
cat <<EOF > $POSTGRES_DIR/pg_hba.conf
# TYPE DATABASE USER ADDRESS METHOD
local all postgres peer

@ -2,7 +2,12 @@
chmod 1777 /tmp
# don't resolve container names to *real* CSC machines
sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
sed -E 's/([[:alnum:]-]+\.)*uwaterloo\.ca//g' /etc/resolv.conf > /tmp/resolv.conf
# remove empty 'search' lines, if we created them
sed -E -i '/^search[[:space:]]*$/d' /tmp/resolv.conf
# also remove the 'rotate' option, since this can cause the Docker DNS server
# to be circumvented
sed -E -i '/^options.*\brotate/d' /tmp/resolv.conf
cp /tmp/resolv.conf /etc/resolv.conf
rm /tmp/resolv.conf
@ -70,6 +75,7 @@ auth_setup() {
# LDAP
apt install -y --no-install-recommends libnss-ldapd
service nslcd stop || true
mkdir -p /etc/ldap
cp .drone/ldap.conf /etc/ldap/ldap.conf
grep -Eq '^map group member uniqueMember$' /etc/nslcd.conf || \
echo 'map group member uniqueMember' >> /etc/nslcd.conf

@ -9,6 +9,8 @@ Make sure your GPG key is in /srv/debian/gpg on potassium-benzoate. See
[here](https://wiki.csclub.uwaterloo.ca/Debian_Repository#Step_1:_Add_to_Uploaders)
for instructions.
Make sure you are in the `csc-mirror` group too.
## Creating the package
Use Docker/Podman to avoid screwing up your main system.
For example, to create a package for bullseye (replace `podman` with `docker` in all instances below if you're using Docker):
@ -58,7 +60,7 @@ podman cp pyceo-packaging:/home/max/repos/pyceo.tar.gz .
(Replace `/home/max/repos` by the directory in the container with the tarball.)
Now upload the tarball to a CSC machine, e.g.
```
scp pyceo.tar.gz mannitol:~/
scp pyceo.tar.gz mannitol:~
```
SSH into that machine and extract the tarball into a separate directory:
```

@ -16,7 +16,7 @@ Docker containers instead, which are much easier to work with than the VM.
First, make sure you create the virtualenv:
```sh
docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
docker run --rm -v "$PWD:$PWD:z" -w "$PWD" python:3.9-bullseye sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
```
Then bring up the containers:
```sh
@ -214,14 +214,23 @@ curl -V
```
Your should see 'SPNEGO' in the 'Features' section.
Here's an example of making a request to an endpoint which writes to LDAP:
Here's an example of making a request to add a user (in the Docker container):
```sh
# Get a Kerberos TGT first
# If you're root, switch to the ctdalek user first
su ctdalek
# Get a Kerberos TGT (password is krb5)
kinit
# Make the request
curl --negotiate -u : --service-name ceod --delegation always \
-d '{"uid":"test_1","cn":"Test One","given_name":"Test","sn":"One","program":"Math","terms":["s2021"]}' \
-X POST http://phosphoric-acid:9987/api/members
# To delete the user:
curl --negotiate -u : --service-name ceod --delegation always \
-X DELETE http://phosphoric-acid:9987/api/members/test_1
# In prod, use the following base URL instead:
# https://phosphoric-acid.csclub.uwaterloo.ca:9987
```
## Packaging

@ -1 +1 @@
1.0.22
1.0.25

@ -28,6 +28,11 @@ def activate():
'Congratulations! Your cloud account has been activated.',
f'You may now login into https://cloud.{base_domain} with your CSC credentials.',
"Make sure to enter 'Members' for the domain (no quotes).",
'',
'Please note that your cloud account will be PERMANENTLY DELETED when',
'your CSC membership expires, so make sure to purchase enough membership',
'terms in advance. You will receive a warning email one week before your',
'cloud account is deleted, so please make sure to check your Junk folder.',
]
for line in lines:
click.echo(line)

@ -4,12 +4,13 @@ from typing import Dict
import click
from zope import component
from ..term_utils import get_terms_for_new_user, get_terms_for_renewal
from ..term_utils import get_terms_for_renewal_for_user
from ..utils import http_post, http_get, http_patch, http_delete, \
get_failed_operations, user_dict_lines, get_adduser_operations
from .utils import handle_stream_response, handle_sync_response, print_lines, \
check_if_in_development
from ceo_common.interfaces import IConfig
from ceo_common.model.Term import get_terms_for_new_user
from ceod.transactions.members import DeleteMemberTransaction
@ -48,7 +49,7 @@ def add(username, cn, given_name, sn, program, num_terms, clubrep, forwarding_ad
sn = result['sn']
if program is None and result.get('program'):
program = result['program']
if forwarding_address is None:
if forwarding_address is None and result.get('mail_local_addresses'):
forwarding_address = result['mail_local_addresses'][0]
if cn is None:
cn = click.prompt('Full name')
@ -155,7 +156,7 @@ def modify(username, login_shell, forwarding_addresses):
@click.option('--clubrep', is_flag=True, default=False,
help='Add non-member terms instead of member terms')
def renew(username, num_terms, clubrep):
terms = get_terms_for_renewal(username, num_terms, clubrep)
terms = get_terms_for_renewal_for_user(username, num_terms, clubrep)
if clubrep:
body = {'non_member_terms': terms}

@ -16,13 +16,22 @@ def positions():
def get():
resp = http_get('/api/positions')
result = handle_sync_response(resp)
print_colon_kv(result.items())
print_colon_kv([
(position, ', '.join(usernames))
for position, usernames in result.items()
])
@positions.command(short_help='Update positions')
def set(**kwargs):
body = {k.replace('_', '-'): v for k, v in kwargs.items()}
print_body = {k: v or '' for k, v in body.items()}
body = {
k.replace('_', '-'): v.replace(' ', '').split(',') if v else None
for k, v in kwargs.items()
}
print_body = {
k: ', '.join(v) if v else ''
for k, v in body.items()
}
click.echo('The positions will be updated:')
print_colon_kv(print_body.items())
click.confirm('Do you want to continue?', abort=True)

@ -1,38 +1,22 @@
from typing import List
from .utils import http_get
from ceo_common.model import Term
from ceo_common.model.Term import get_terms_for_renewal
import ceo.cli.utils as cli_utils
import ceo.tui.utils as tui_utils
# Had to put these in a separate file to avoid a circular import.
def get_terms_for_new_user(num_terms: int) -> List[str]:
current_term = Term.current()
terms = [current_term + i for i in range(num_terms)]
return list(map(str, terms))
def get_terms_for_renewal(
def get_terms_for_renewal_for_user(
username: str, num_terms: int, clubrep: bool, tui_controller=None,
) -> List[str]:
resp = http_get('/api/members/' + username)
# FIXME: this is ugly, we shouldn't need a hacky if statement like this
if tui_controller is None:
result = cli_utils.handle_sync_response(resp)
else:
result = tui_utils.handle_sync_response(resp, tui_controller)
max_term = None
current_term = Term.current()
if clubrep and 'non_member_terms' in result:
max_term = max(Term(s) for s in result['non_member_terms'])
elif not clubrep and 'terms' in result:
max_term = max(Term(s) for s in result['terms'])
if max_term is not None and max_term >= current_term:
next_term = max_term + 1
if clubrep:
return get_terms_for_renewal(result.get('non_member_terms'), num_terms)
else:
next_term = Term.current()
terms = [next_term + i for i in range(num_terms)]
return list(map(str, terms))
return get_terms_for_renewal(result.get('terms'), num_terms)

@ -3,9 +3,9 @@ from threading import Thread
from ...utils import http_get
from .Controller import Controller
from .AddUserTransactionController import AddUserTransactionController
import ceo.term_utils as term_utils
from ceo.tui.models import TransactionModel
from ceo.tui.views import AddUserConfirmationView, TransactionView
from ceo_common.model.Term import get_terms_for_new_user
from ceod.transactions.members import AddMemberTransaction
@ -26,7 +26,7 @@ class AddUserController(Controller):
body['program'] = self.model.program
if self.model.forwarding_address:
body['forwarding_addresses'] = [self.model.forwarding_address]
new_terms = term_utils.get_terms_for_new_user(self.model.num_terms)
new_terms = get_terms_for_new_user(self.model.num_terms)
if self.model.membership_type == 'club_rep':
body['non_member_terms'] = new_terms
else:
@ -106,5 +106,5 @@ class AddUserController(Controller):
self.model.first_name = data.get('given_name', '')
self.model.last_name = data.get('sn', '')
self.model.program = data.get('program', '')
self.model.forwarding_address = data.get('mail_local_addresses', [''])[0]
self.model.forwarding_address = (data.get('mail_local_addresses') or [''])[0]
self.app.run_in_main_loop(self._on_lookup_user_success)

@ -19,8 +19,8 @@ class GetPositionsController(Controller):
positions = tui_utils.handle_sync_response(resp, self)
except Controller.RequestFailed:
return
for pos, username in positions.items():
self.model.positions[pos] = username
for pos, usernames in positions.items():
self.model.positions[pos] = ','.join(usernames)
def target():
self.view.flash_text.set_text('')

@ -28,7 +28,7 @@ class RenewUserController(SyncRequestController):
def _get_next_terms(self):
try:
self.model.new_terms = term_utils.get_terms_for_renewal(
self.model.new_terms = term_utils.get_terms_for_renewal_for_user(
self.model.username,
self.model.num_terms,
self.model.membership_type == 'club_rep',

@ -17,7 +17,7 @@ class SetPositionsController(Controller):
body = {}
for pos, field in self.view.position_fields.items():
if field.edit_text != '':
body[pos] = field.edit_text
body[pos] = field.edit_text.replace(' ', '').split(',')
model = TransactionModel(
UpdateMemberPositionsTransaction.operations,
'POST', '/api/positions', json=body
@ -37,8 +37,8 @@ class SetPositionsController(Controller):
positions = tui_utils.handle_sync_response(resp, self)
except Controller.RequestFailed:
return
for pos, username in positions.items():
self.model.positions[pos] = username
for pos, usernames in positions.items():
self.model.positions[pos] = ','.join(usernames)
def target():
self.view.flash_text.set_text('')

@ -53,6 +53,10 @@ class AddUserView(ColumnView):
urwid.Text('Program:', align='right'),
self.program_edit
),
(
urwid.Text('Forwarding address:', align='right'),
self.forwarding_address_edit
),
(
urwid.Text('Number of terms:', align='right'),
self.num_terms_edit

@ -87,6 +87,9 @@ def space_colon_kv(pairs: List[Tuple[str, str]]) -> List[str]:
maxlen = max(len(key) for key, val in pairs)
for key, val in pairs:
if key != '':
if not val:
lines.append(key + ':')
continue
prefix = key + ': '
else:
# assume this is a continuation from the previous line
@ -136,6 +139,8 @@ def user_dict_kv(d: Dict) -> List[Tuple[str]]:
pairs.append(('non-member terms', ','.join(_terms)))
if 'password' in d:
pairs.append(('password', d['password']))
if 'groups' in d:
pairs.append(('groups', ','.join(d['groups'])))
return pairs

@ -19,8 +19,7 @@ class ICloudStackService(Interface):
The dict is mapping of usernames to account IDs.
"""
def delete_account(account_id: str):
def delete_account(username: str, account_id: str):
"""
Delete the given CloudStack account.
Note that a CloudStack account ID must be given, not a username.
"""

@ -33,6 +33,11 @@ class ILDAPService(Interface):
A new UID and GID will be generated and returned in the new user.
"""
def get_groups_for_user(username: str) -> List[str]:
"""
Get a list of the groups to which the user belongs.
"""
def remove_user(user: IUser):
"""Remove this user from the database."""

@ -6,7 +6,7 @@ from requests_gssapi import HTTPSPNEGOAuth
from zope import component
from zope.interface import implementer
from ceo_common.interfaces import IConfig, IHTTPClient
from ceo_common.interfaces import IConfig, IHTTPClient, IKerberosService
@implementer(IHTTPClient)
@ -40,10 +40,18 @@ class HTTPClient:
'opportunistic_auth': True,
'target_name': gssapi.Name('ceod/' + host),
}
if flask.has_request_context() and 'client_token' in g:
if flask.has_request_context():
# This is reached when we are the server and the client has
# forwarded their credentials to us.
spnego_kwargs['creds'] = gssapi.Credentials(token=g.client_token)
token = None
if g.get('need_admin_creds', False):
# Some Kerberos bindings in some programming languages can't
# perform delegation, so use the admin creds here.
token = component.getUtility(IKerberosService).get_admin_creds_token()
elif 'client_token' in g:
token = g.client_token
if token is not None:
spnego_kwargs['creds'] = gssapi.Credentials(token=token)
elif delegate:
# This is reached when we are the client and we want to
# forward our credentials to the server.

@ -1,4 +1,5 @@
import datetime
from typing import List, Union
import ceo_common.utils as utils
@ -81,3 +82,39 @@ class Term:
month = self.seasons.index(c) * 4 + 1
day = 1
return datetime.datetime(year, month, day)
# Utility functions
def get_terms_for_new_user(num_terms: int) -> List[str]:
current_term = Term.current()
terms = [current_term + i for i in range(num_terms)]
return list(map(str, terms))
def get_terms_for_renewal(
existing_terms: Union[List[str], None],
num_terms: int,
) -> List[str]:
"""Calculates the terms for which a member or club rep should be renewed.
:param terms: The existing terms for the user being renewed. If the user
is being renewed as a regular member, these should be the
member terms. If they are being renewed as a club rep, these
should be the non-member terms.
This may be None if the user does not have any terms of the
appropriate type (an empty list is also acceptable).
:param num_terms: The number of terms for which the user is being renewed.
"""
max_term = None
current_term = Term.current()
if existing_terms:
max_term = max(map(Term, existing_terms))
if max_term is not None and max_term >= current_term:
next_term = max_term + 1
else:
next_term = current_term
terms = [next_term + i for i in range(num_terms)]
return list(map(str, terms))

@ -1,13 +1,14 @@
from flask import Blueprint, g, request
from flask import Blueprint, request
from flask.json import jsonify
from zope import component
from .utils import authz_restrict_to_staff, authz_restrict_to_syscom, \
user_is_in_group, requires_authentication_no_realm, \
user_is_in_group, requires_authentication_no_realm, requires_admin_creds, \
create_streaming_response, development_only, is_truthy
from ceo_common.errors import BadRequest, UserAlreadySubscribedError, UserNotSubscribedError
from ceo_common.interfaces import ILDAPService, IConfig, IMailService
from ceo_common.logger_factory import logger_factory
from ceo_common.model.Term import get_terms_for_new_user, get_terms_for_renewal
from ceod.transactions.members import (
AddMemberTransaction,
ModifyMemberTransaction,
@ -20,20 +21,29 @@ logger = logger_factory(__name__)
@bp.route('/', methods=['POST'], strict_slashes=False)
@requires_admin_creds
@authz_restrict_to_staff
def create_user():
# We need to use the admin creds here because office members may not
# directly create new LDAP records.
body = request.get_json(force=True)
terms = body.get('terms')
non_member_terms = body.get('non_member_terms')
if (terms and non_member_terms) or not (terms or non_member_terms):
raise BadRequest('Must specify either terms or non-member terms')
if type(terms) is int:
terms = get_terms_for_new_user(terms)
elif type(non_member_terms) is int:
non_member_terms = get_terms_for_new_user(non_member_terms)
for attr in ['uid', 'cn', 'given_name', 'sn']:
if not body.get(attr):
raise BadRequest(f"Attribute '{attr}' is missing or empty")
# We need to use the admin creds here because office members may not
# directly create new LDAP records.
g.need_admin_creds = True
if terms:
logger.info(f"Creating member {body['uid']} for terms {terms}")
else:
logger.info(f"Creating club rep {body['uid']} for non-member terms {non_member_terms}")
txn = AddMemberTransaction(
uid=body['uid'],
@ -61,7 +71,9 @@ def get_user(auth_user: str, username: str):
get_forwarding_addresses = True
ldap_srv = component.getUtility(ILDAPService)
user = ldap_srv.get_user(username)
return user.to_dict(get_forwarding_addresses)
user_dict = user.to_dict(get_forwarding_addresses)
user_dict['groups'] = ldap_srv.get_groups_for_user(username)
return user_dict
@bp.route('/<username>', methods=['PATCH'])
@ -81,40 +93,48 @@ def patch_user(auth_user: str, username: str):
@bp.route('/<username>/renew', methods=['POST'])
@requires_admin_creds
@authz_restrict_to_staff
def renew_user(username: str):
# We need to use the admin creds here because office members should
# not be able to directly modify the shadowExpire field; this could
# prevent syscom members from logging into the machines.
body = request.get_json(force=True)
terms = body.get('terms')
non_member_terms = body.get('non_member_terms')
if (terms and non_member_terms) or not (terms or non_member_terms):
raise BadRequest('Must specify either terms or non-member terms')
# We need to use the admin creds here because office members should
# not be able to directly modify the shadowExpire field; this could
# prevent syscom members from logging into the machines.
g.need_admin_creds = True
ldap_srv = component.getUtility(ILDAPService)
cfg = component.getUtility(IConfig)
user = ldap_srv.get_user(username)
member_list = cfg.get('mailman3_new_member_list')
if type(terms) is int:
terms = get_terms_for_renewal(user.terms, terms)
elif type(non_member_terms) is int:
non_member_terms = get_terms_for_renewal(user.non_member_terms, non_member_terms)
def unexpire(user):
if user.shadowExpire:
user.set_expired(False)
try:
user.subscribe_to_mailing_list(member_list)
logger.debug(f'Subscribed {user.uid} to {member_list}')
except UserAlreadySubscribedError:
pass
logger.debug(f'{user.uid} is already subscribed to {member_list}')
if body.get('terms'):
user.add_terms(body['terms'])
if terms:
logger.info(f"Renewing member {username} for terms {terms}")
user.add_terms(terms)
unexpire(user)
return {'terms_added': body['terms']}
elif body.get('non_member_terms'):
user.add_non_member_terms(body['non_member_terms'])
return {'terms_added': terms}
elif non_member_terms:
logger.info(f"Renewing club rep {username} for non-member terms {non_member_terms}")
user.add_non_member_terms(non_member_terms)
unexpire(user)
return {'non_member_terms_added': body['non_member_terms']}
return {'non_member_terms_added': non_member_terms}
else:
raise BadRequest('Must specify either terms or non-member terms')
@ -149,11 +169,13 @@ def expire_users():
if not dry_run:
for member in members:
logger.info(f'Expiring {member.uid}')
member.set_expired(True)
try:
member.unsubscribe_from_mailing_list(member_list)
logger.debug(f'Unsubscribed {member.uid} from {member_list}')
except UserNotSubscribedError:
pass
logger.debug(f'{member.uid} is already unsubscribed from {member_list}')
return jsonify([member.uid for member in members])

@ -2,6 +2,7 @@ from flask import Blueprint, request
from zope import component
from .utils import authz_restrict_to_syscom, create_streaming_response
from ceo_common.errors import BadRequest
from ceo_common.interfaces import ILDAPService, IConfig
from ceod.transactions.members import UpdateMemberPositionsTransaction
@ -15,7 +16,9 @@ def get_positions():
positions = {}
for user in ldap_srv.get_users_with_positions():
for position in user.positions:
positions[position] = user.uid
if position not in positions:
positions[position] = []
positions[position].append(user.uid)
return positions
@ -29,23 +32,31 @@ def update_positions():
required = cfg.get('positions_required')
available = cfg.get('positions_available')
# remove falsy values
body = {
positions: username for positions, username in body.items()
if username
}
for position in body.keys():
# remove falsy values and parse multiple users in each position
# Example: "user1,user2, user3" -> ["user1","user2","user3"]
position_to_usernames = {}
for position, usernames in body.items():
if not usernames:
continue
if type(usernames) is list:
position_to_usernames[position] = usernames
elif type(usernames) is str:
position_to_usernames[position] = usernames.replace(' ', '').split(',')
else:
raise BadRequest('usernames must be a list or comma-separated string')
# check for duplicates (i.e. one username specified twice in the same list)
for usernames in position_to_usernames.values():
if len(usernames) != len(set(usernames)):
raise BadRequest('username may only be specified at most once for a position')
for position in position_to_usernames.keys():
if position not in available:
return {
'error': f'unknown position: {position}'
}, 400
raise BadRequest(f'unknown position: {position}')
for position in required:
if position not in body:
return {
'error': f'missing required position: {position}'
}, 400
if position not in position_to_usernames:
raise BadRequest(f'missing required position: {position}')
txn = UpdateMemberPositionsTransaction(body)
txn = UpdateMemberPositionsTransaction(position_to_usernames)
return create_streaming_response(txn)

@ -1,12 +1,9 @@
import functools
import grp
import json
import os
import pwd
import traceback
from typing import Callable, List
from flask import current_app, stream_with_context
from flask import current_app, g, stream_with_context
from zope import component
from .spnego import requires_authentication
@ -40,9 +37,24 @@ def requires_authentication_no_realm(f: Callable) -> Callable:
return wrapper
def user_is_in_group(user: str, group: str) -> bool:
"""Returns True if `user` is in `group`, False otherwise."""
return user in grp.getgrnam(group).gr_mem
def requires_admin_creds(f: Callable) -> Callable:
"""
Forces the next LDAP connection to use the admin Kerberos credentials.
This must be used BEFORE any of the authz decorators, since those
may require an LDAP connection, which will get cached for later use.
"""
@functools.wraps(f)
def wrapper(*args, **kwargs):
g.need_admin_creds = True
return f(*args, **kwargs)
return wrapper
def user_is_in_group(username: str, group_name: str) -> bool:
"""Returns True if `username` is in `group_name`, False otherwise."""
ldap_srv = component.getUtility(ILDAPService)
group = ldap_srv.get_group(group_name)
return username in group.members
def authz_restrict_to_groups(f: Callable, allowed_groups: List[str]) -> Callable:
@ -51,8 +63,6 @@ def authz_restrict_to_groups(f: Callable, allowed_groups: List[str]) -> Callable
specified groups.
"""
allowed_group_ids = [grp.getgrnam(g).gr_gid for g in allowed_groups]
@requires_authentication_no_realm
@functools.wraps(f)
def wrapper(_username: str, *args, **kwargs):
@ -62,15 +72,14 @@ def authz_restrict_to_groups(f: Callable, allowed_groups: List[str]) -> Callable
if username.startswith('ceod/'):
# ceod services are always allowed to make internal calls
return f(*args, **kwargs)
for gid in os.getgrouplist(username, pwd.getpwnam(username).pw_gid):
if gid in allowed_group_ids:
ldap_srv = component.getUtility(ILDAPService)
for group_name in ldap_srv.get_groups_for_user(username):
if group_name in allowed_groups:
return f(*args, **kwargs)
logger.debug(
f"User '{username}' denied since they are not in one of {allowed_groups}"
)
return {
'error': f'You must be in one of {allowed_groups}'
}, 403
return {'error': f'You must be in one of {allowed_groups}'}, 403
return wrapper

@ -97,7 +97,7 @@ class CloudResourceManager:
resources = accounts[username]['resources']
if 'cloudstack' in resources:
account_id = accounts[username]['cloudstack_account_id']
cloudstack_srv.delete_account(account_id)
cloudstack_srv.delete_account(username, account_id)
if 'vhost' in resources:
vhost_mgr.delete_all_vhosts_for_user(username)
if 'k8s' in resources:

@ -76,7 +76,8 @@ class CloudStackService:
for account in d['account']
}
def delete_account(self, account_id: str):
def delete_account(self, username: str, account_id: str):
logger.info(f'Deleting CloudStack account for {username}')
url = self._create_url({
'command': 'deleteAccount',
'id': account_id,

@ -84,16 +84,19 @@ class ClubWebHostingService:
logger.debug('Reloading Apache')
self._run(['systemctl', 'reload', 'apache2'])
# This requires the APACHE_CONFIG_CRON environment variable to be
# set to 1 (e.g. in a systemd drop-in)
# See /etc/apache2/.git/hooks/pre-commit on caffeine
def _git_commit(self):
if not os.path.isdir(os.path.join(self.apache_dir, '.git')):
logger.debug('No git folder found in Apache directory')
return
logger.debug('Committing changes to git repository')
self._run(['git', 'add', APACHE_DISABLED_CLUBS_FILE], cwd=self.apache_dir)
self._run(['git', 'commit', '-m', '[ceo] disable club websites'], cwd=self.apache_dir)
self._run(
['git', 'add', APACHE_DISABLED_CLUBS_FILE],
cwd=self.apache_dir)
# See /etc/apache2/.git/hooks/pre-commit on caffeine
self._run(
['git', 'commit', '-m', '[ceo] disable club websites'],
cwd=self.apache_dir,
env={**os.environ, 'APACHE_CONFIG_CRON': '1'})
def commit(self):
if not self.made_at_least_one_change:
@ -112,12 +115,13 @@ class ClubWebHostingService:
directive_paths = self.aug.match(f'/files/etc/apache2/sites-available/{filename}/VirtualHost/directive')
for directive_path in directive_paths:
directive = self.aug.get(directive_path)
directive_value = self.aug.get(directive_path + '/arg')
if directive == 'DocumentRoot':
directive_value = self.aug.get(directive_path + '/arg')
match = APACHE_USERDIR_RE.match(directive_value)
if match is not None:
club_name = match.group('club_name')
elif directive == 'ServerAdmin':
directive_value = self.aug.get(directive_path + '/arg')
club_email = directive_value
if club_name is not None:
self.clubs[club_name]['email'] = club_email
@ -157,12 +161,20 @@ class ClubWebHostingService:
def _site_uses_php(self, club_name: str) -> bool:
www = f'{self.clubs_home}/{club_name}/www'
if os.path.isdir(www):
if not os.path.isdir(www):
return False
try:
# We're just going to look one level deep; that should be good enough.
for filename in os.listdir(www):
filepath = os.path.join(www, filename)
if os.path.isfile(filepath) and filename.endswith('.php'):
return True
filenames = os.listdir(www)
except os.error:
# If we're unable to read the directory (e.g. permissions error),
# then this means that the Apache user (www-data) can't read it either.
# So we can just return False here.
return False
for filename in filenames:
filepath = os.path.join(www, filename)
if os.path.isfile(filepath) and filename.endswith('.php'):
return True
return False
# This method needs to be called from within a transaction (uses self.clubs)
@ -224,7 +236,7 @@ class ClubWebHostingService:
# STEP 2: send emails to clubs whose websites were disabled
clubs_who_were_not_notified = set()
for club_name in clubs_to_disable:
address = clubs_info['email']
address = clubs_info[club_name]['email']
if address is None:
clubs_who_were_not_notified.add(club_name)
continue

@ -7,6 +7,9 @@ from zope.interface import implementer
from ceo_common.errors import UserNotFoundError
from ceo_common.interfaces import IContainerRegistryService, IConfig
from ceo_common.logger_factory import logger_factory
logger = logger_factory(__name__)
@implementer(IContainerRegistryService)
@ -70,6 +73,7 @@ class ContainerRegistryService:
resp.raise_for_status()
def delete_project_for_user(self, username: str):
logger.info(f'Deleting Harbor project for {username}')
# Delete all of the repositories inside the project first
resp = self._http_get(f'/projects/{username}/repositories')
if resp.status_code == 403:

@ -11,6 +11,9 @@ from zope import component
from zope.interface import implementer
from ceo_common.interfaces import IConfig, IKubernetesService
from ceo_common.logger_factory import logger_factory
logger = logger_factory(__name__)
@implementer(IKubernetesService)
@ -100,6 +103,7 @@ class KubernetesService:
return body
def delete_account(self, username: str):
logger.info(f'Deleting Kubernetes namespace for {username}')
namespace = self._get_namespace(username)
# don't check exit code because namespace might not exist
self._run(['kubectl', 'delete', 'namespace', namespace], check=False)

@ -98,6 +98,13 @@ class LDAPService:
entry = self._get_readable_entry_for_group(conn, cn)
return Group.deserialize_from_ldap(entry)
def get_groups_for_user(self, username: str) -> List[str]:
conn = self._get_ldap_conn()
conn.search(self.ldap_groups_base,
f'(uniqueMember={self.uid_to_dn(username)})',
attributes=['cn'])
return sorted([entry.cn.value for entry in conn.entries])
def get_display_info_for_users(self, usernames: List[str]) -> List[Dict[str, str]]:
if not usernames:
return []
@ -105,14 +112,14 @@ class LDAPService:
filter = '(|' + ''.join([f'(uid={uid})' for uid in usernames]) + ')'
attributes = ['uid', 'cn', 'program']
conn.search(self.ldap_users_base, filter, attributes=attributes)
return [
return sorted([
{
'uid': entry.uid.value,
'cn': entry.cn.value,
'program': entry.program.value or 'Unknown',
}
for entry in conn.entries
]
], key=lambda member: member['uid'])
def get_users_with_positions(self) -> List[IUser]:
conn = self._get_ldap_conn()
@ -309,12 +316,12 @@ class LDAPService:
self,
dry_run: bool = False,
members: Union[List[str], None] = None,
uwldap_batch_size: int = 10,
uwldap_batch_size: int = 100,
):
if members:
filter = '(|' + ''.join([f'(uid={uid})' for uid in members]) + ')'
else:
filter = '(objectClass=*)'
filter = '(objectClass=member)'
conn = self._get_ldap_conn()
conn.search(
self.ldap_users_base, filter, attributes=['uid', 'program'])
@ -329,12 +336,17 @@ class LDAPService:
batch_uids = uids[i:i + uwldap_batch_size]
batch_uw_programs = uwldap_srv.get_programs_for_users(batch_uids)
uw_programs.extend(batch_uw_programs)
# uw_programs[i] will be None if the 'ou' attribute was not
# present in UWLDAP, or if no UWLDAP entry was found at all
for i, uw_program in enumerate(uw_programs):
if uw_program in (None, 'expired', 'orphaned'):
# If the UWLDAP record is orphaned, nonexistent, or missing
# data, assume that the member graduated
uw_programs[i] = 'Alumni'
users_to_change = [
(uids[i], csc_programs[i], uw_programs[i])
for i in range(len(uids))
if csc_programs[i] != uw_programs[i] and (
uw_programs[i] not in (None, 'expired', 'orphaned')
)
if csc_programs[i] != uw_programs[i]
]
if dry_run:
return users_to_change

@ -86,7 +86,6 @@ class User:
'is_club': self.is_club(),
'is_club_rep': self.is_club_rep,
'program': self.program or 'Unknown',
'shadowExpire': self.shadowExpire,
}
if self.sn and self.given_name:
data['sn'] = self.sn
@ -103,6 +102,8 @@ class User:
data['mail_local_addresses'] = self.mail_local_addresses
if get_forwarding_addresses:
data['forwarding_addresses'] = self.get_forwarding_addresses()
if self.shadowExpire:
data['shadow_expire'] = self.shadowExpire
return data
def __repr__(self) -> str:
@ -169,7 +170,7 @@ class User:
is_club_rep=attrs.get('isClubRep', [False])[0],
is_club=('club' in attrs['objectClass']),
is_member_or_club_rep=('member' in attrs['objectClass']),
shadowExpire=attrs.get('shadowExpire'),
shadowExpire=attrs.get('shadowExpire', [None])[0],
ldap3_entry=entry,
)

@ -4,7 +4,7 @@ import os
import re
import shutil
import subprocess
from typing import List, Dict, Tuple
from typing import List, Dict, Tuple, Union
import jinja2
from zope import component
@ -53,6 +53,7 @@ class VHostManager:
self.max_vhosts_per_account = cfg.get('cloud vhosts_max_vhosts_per_account')
self.vhost_ip_min = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_min'))
self.vhost_ip_max = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_max'))
self.reload_web_server_cmd = cfg.get('cloud vhosts_reload_web_server_cmd')
self.acme_challenge_dir = cfg.get('cloud vhosts_acme_challenge_dir')
self.acme_dir = '/root/.acme.sh'
@ -82,12 +83,12 @@ class VHostManager:
"""Return a list of all vhost files for this user."""