Add group lookup functionality (#88)

note: **I am unaware of best practices** but I tried my best to keep changes consistent with the codebase

feedback would be much appreciated

notable changes:
**new api endpoint**: `/groups/search` -- I moved searching into the api so it could be used in tui and cli, also seemed like a good idea to keep the json response as small as possible
**tui searching** -- at first I wanted to make this realtime interactable, but the work required seemed inappropriate to a feature I am assuming will only be used sparingly

Co-authored-by: Daniel Sun <dandancool@github.com>
Co-authored-by: Daniel Sun <d6sun@uwaterloo.ca>
Reviewed-on: #88
Reviewed-by: Max Erenberg <merenber@csclub.uwaterloo.ca>
Co-authored-by: Daniel Sun <d6sun@csclub.uwaterloo.ca>
Co-committed-by: Daniel Sun <d6sun@csclub.uwaterloo.ca>

.drone.yml

@@ -5,7 +5,7 @@ name: default
 steps:
   # use the step name to mock out the gethostname() call in our tests
 - name: phosphoric-acid
-  image: python:3.7-buster
+  image: python:3.9-bullseye
   # unfortunately we have to do everything in one step because there's no
   # way to share system packages between steps
   commands:
@@ -25,12 +25,12 @@ steps:
 
 services:
   - name: auth1
-    image: debian:buster
+    image: debian:bullseye
     commands:
       - .drone/auth1-setup.sh
       - sleep infinity
   - name: coffee
-    image: debian:buster
+    image: debian:bullseye
     commands:
       - .drone/coffee-setup.sh
       - sleep infinity

.drone/auth1-setup.sh

@@ -28,7 +28,6 @@ killall slapd || true
 service nslcd stop || true
 rm -rf /etc/ldap/slapd.d
 rm /var/lib/ldap/*
-cp /usr/share/slapd/DB_CONFIG /var/lib/ldap/DB_CONFIG
 cp .drone/slapd.conf /etc/ldap/slapd.conf
 cp .drone/ldap.conf /etc/ldap/ldap.conf
 cp /usr/share/doc/sudo-ldap/schema.OpenLDAP /etc/ldap/schema/sudo.schema

.drone/coffee-setup.sh

@@ -11,9 +11,9 @@ add_fqdn_to_hosts $(get_ip_addr auth1) auth1
 apt install --no-install-recommends -y default-mysql-server postgresql
 
 # MYSQL
-service mysql stop
-sed -E -i 's/^(bind-address[[:space:]]+= 127.0.0.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf
-service mysql start
+service mariadb stop
+sed -E -i 's/^(bind-address[[:space:]]+= 127\.0\.0\.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf
+service mariadb start
 cat <<EOF | mysql
 CREATE USER IF NOT EXISTS 'mysql' IDENTIFIED BY 'mysql';
 GRANT ALL PRIVILEGES ON *.* TO 'mysql' WITH GRANT OPTION;
@@ -21,7 +21,7 @@ EOF
 
 # POSTGRESQL
 service postgresql stop
-POSTGRES_DIR=/etc/postgresql/11/main
+POSTGRES_DIR=/etc/postgresql/*/main
 cat <<EOF > $POSTGRES_DIR/pg_hba.conf
 # TYPE  DATABASE        USER            ADDRESS                 METHOD
 local   all             postgres                                peer

.drone/common.sh

@@ -2,7 +2,12 @@
 chmod 1777 /tmp
 
 # don't resolve container names to *real* CSC machines
-sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
+sed -E 's/([[:alnum:]-]+\.)*uwaterloo\.ca//g' /etc/resolv.conf > /tmp/resolv.conf
+# remove empty 'search' lines, if we created them
+sed -E -i '/^search[[:space:]]*$/d' /tmp/resolv.conf
+# also remove the 'rotate' option, since this can cause the Docker DNS server
+# to be circumvented
+sed -E -i '/^options.*\brotate/d' /tmp/resolv.conf
 cp /tmp/resolv.conf /etc/resolv.conf
 rm /tmp/resolv.conf
 
@@ -70,6 +75,7 @@ auth_setup() {
     # LDAP
     apt install -y --no-install-recommends libnss-ldapd
     service nslcd stop || true
+    mkdir -p /etc/ldap
     cp .drone/ldap.conf /etc/ldap/ldap.conf
     grep -Eq '^map group member uniqueMember$' /etc/nslcd.conf || \
         echo 'map group member uniqueMember' >> /etc/nslcd.conf

PACKAGING.md

@@ -9,6 +9,8 @@ Make sure your GPG key is in /srv/debian/gpg on potassium-benzoate. See
 [here](https://wiki.csclub.uwaterloo.ca/Debian_Repository#Step_1:_Add_to_Uploaders)
 for instructions.
 
+Make sure you are in the `csc-mirror` group too.
+
 ## Creating the package
 Use Docker/Podman to avoid screwing up your main system.
 For example, to create a package for bullseye (replace `podman` with `docker` in all instances below if you're using Docker):
@@ -58,7 +60,7 @@ podman cp pyceo-packaging:/home/max/repos/pyceo.tar.gz .
 (Replace `/home/max/repos` by the directory in the container with the tarball.)
 Now upload the tarball to a CSC machine, e.g.
 ```
-scp pyceo.tar.gz mannitol:~/
+scp pyceo.tar.gz mannitol:~
 ```
 SSH into that machine and extract the tarball into a separate directory:
 ```

README.md

@@ -16,7 +16,7 @@ Docker containers instead, which are much easier to work with than the VM.
 
 First, make sure you create the virtualenv:
 ```sh
-docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
+docker run --rm -v "$PWD:$PWD:z" -w "$PWD" python:3.9-bullseye sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
 ```
 Then bring up the containers:
 ```sh
@@ -214,14 +214,23 @@ curl -V
 ```
 Your should see 'SPNEGO' in the 'Features' section.
 
-Here's an example of making a request to an endpoint which writes to LDAP:
+Here's an example of making a request to add a user (in the Docker container):
 ```sh
-# Get a Kerberos TGT first
+# If you're root, switch to the ctdalek user first
+su ctdalek
+# Get a Kerberos TGT (password is krb5)
 kinit
 # Make the request
 curl --negotiate -u : --service-name ceod --delegation always \
     -d '{"uid":"test_1","cn":"Test One","given_name":"Test","sn":"One","program":"Math","terms":["s2021"]}' \
     -X POST http://phosphoric-acid:9987/api/members
+
+# To delete the user:
+curl --negotiate -u : --service-name ceod --delegation always \
+    -X DELETE http://phosphoric-acid:9987/api/members/test_1
+
+# In prod, use the following base URL instead:
+# https://phosphoric-acid.csclub.uwaterloo.ca:9987
 ```
 
 ## Packaging

VERSION.txt

@@ -1 +1 @@
-1.0.21
+1.0.26

ceo/cli/cloud.py

@@ -28,6 +28,11 @@ def activate():
         'Congratulations! Your cloud account has been activated.',
         f'You may now login into https://cloud.{base_domain} with your CSC credentials.',
         "Make sure to enter 'Members' for the domain (no quotes).",
+        '',
+        'Please note that your cloud account will be PERMANENTLY DELETED when',
+        'your CSC membership expires, so make sure to purchase enough membership',
+        'terms in advance. You will receive a warning email one week before your',
+        'cloud account is deleted, so please make sure to check your Junk folder.',
     ]
     for line in lines:
         click.echo(line)

ceo/cli/groups.py

@@ -149,3 +149,15 @@ def delete(group_name):
     click.confirm(f"Are you sure you want to delete {group_name}?", abort=True)
     resp = http_delete(f'/api/groups/{group_name}')
     handle_stream_response(resp, DeleteGroupTransaction.operations)
+
+
+@groups.command(short_help='Search for groups')
+@click.argument('query')
+@click.option('--count', default=10, help='number of results to show')
+def search(query, count):
+    check_if_in_development()
+    resp = http_get(f'/api/groups/search/{query}/{count}')
+    result = handle_sync_response(resp)
+    for cn in result:
+        if cn != "":
+            click.echo(cn)

ceo/cli/members.py

@@ -4,12 +4,13 @@ from typing import Dict
 import click
 from zope import component
 
-from ..term_utils import get_terms_for_new_user, get_terms_for_renewal
+from ..term_utils import get_terms_for_renewal_for_user
 from ..utils import http_post, http_get, http_patch, http_delete, \
     get_failed_operations, user_dict_lines, get_adduser_operations
 from .utils import handle_stream_response, handle_sync_response, print_lines, \
     check_if_in_development
 from ceo_common.interfaces import IConfig
+from ceo_common.model.Term import get_terms_for_new_user
 from ceod.transactions.members import DeleteMemberTransaction
 
 
@@ -48,7 +49,7 @@ def add(username, cn, given_name, sn, program, num_terms, clubrep, forwarding_ad
             sn = result['sn']
         if program is None and result.get('program'):
             program = result['program']
-        if forwarding_address is None:
+        if forwarding_address is None and result.get('mail_local_addresses'):
             forwarding_address = result['mail_local_addresses'][0]
     if cn is None:
         cn = click.prompt('Full name')
@@ -155,7 +156,7 @@ def modify(username, login_shell, forwarding_addresses):
 @click.option('--clubrep', is_flag=True, default=False,
               help='Add non-member terms instead of member terms')
 def renew(username, num_terms, clubrep):
-    terms = get_terms_for_renewal(username, num_terms, clubrep)
+    terms = get_terms_for_renewal_for_user(username, num_terms, clubrep)
 
     if clubrep:
         body = {'non_member_terms': terms}

ceo/cli/positions.py

@@ -16,13 +16,22 @@ def positions():
 def get():
     resp = http_get('/api/positions')
     result = handle_sync_response(resp)
-    print_colon_kv(result.items())
+    print_colon_kv([
+        (position, ', '.join(usernames))
+        for position, usernames in result.items()
+    ])
 
 
 @positions.command(short_help='Update positions')
 def set(**kwargs):
-    body = {k.replace('_', '-'): v for k, v in kwargs.items()}
-    print_body = {k: v or '' for k, v in body.items()}
+    body = {
+        k.replace('_', '-'): v.replace(' ', '').split(',') if v else None
+        for k, v in kwargs.items()
+    }
+    print_body = {
+        k: ', '.join(v) if v else ''
+        for k, v in body.items()
+    }
     click.echo('The positions will be updated:')
     print_colon_kv(print_body.items())
     click.confirm('Do you want to continue?', abort=True)

ceo/term_utils.py

@@ -1,38 +1,22 @@
 from typing import List
 
 from .utils import http_get
-from ceo_common.model import Term
+from ceo_common.model.Term import get_terms_for_renewal
 import ceo.cli.utils as cli_utils
 import ceo.tui.utils as tui_utils
 
-# Had to put these in a separate file to avoid a circular import.
 
-
-def get_terms_for_new_user(num_terms: int) -> List[str]:
-    current_term = Term.current()
-    terms = [current_term + i for i in range(num_terms)]
-    return list(map(str, terms))
-
-
-def get_terms_for_renewal(
+def get_terms_for_renewal_for_user(
     username: str, num_terms: int, clubrep: bool, tui_controller=None,
 ) -> List[str]:
     resp = http_get('/api/members/' + username)
+    # FIXME: this is ugly, we shouldn't need a hacky if statement like this
     if tui_controller is None:
         result = cli_utils.handle_sync_response(resp)
     else:
         result = tui_utils.handle_sync_response(resp, tui_controller)
-    max_term = None
-    current_term = Term.current()
-    if clubrep and 'non_member_terms' in result:
-        max_term = max(Term(s) for s in result['non_member_terms'])
-    elif not clubrep and 'terms' in result:
-        max_term = max(Term(s) for s in result['terms'])
 
-    if max_term is not None and max_term >= current_term:
-        next_term = max_term + 1
+    if clubrep:
+        return get_terms_for_renewal(result.get('non_member_terms'), num_terms)
     else:
-        next_term = Term.current()
-
-    terms = [next_term + i for i in range(num_terms)]
-    return list(map(str, terms))
+        return get_terms_for_renewal(result.get('terms'), num_terms)

ceo/tui/controllers/AddUserController.py

@@ -3,9 +3,9 @@ from threading import Thread
 from ...utils import http_get
 from .Controller import Controller
 from .AddUserTransactionController import AddUserTransactionController
-import ceo.term_utils as term_utils
 from ceo.tui.models import TransactionModel
 from ceo.tui.views import AddUserConfirmationView, TransactionView
+from ceo_common.model.Term import get_terms_for_new_user
 from ceod.transactions.members import AddMemberTransaction
 
 
@@ -26,7 +26,7 @@ class AddUserController(Controller):
             body['program'] = self.model.program
         if self.model.forwarding_address:
             body['forwarding_addresses'] = [self.model.forwarding_address]
-        new_terms = term_utils.get_terms_for_new_user(self.model.num_terms)
+        new_terms = get_terms_for_new_user(self.model.num_terms)
         if self.model.membership_type == 'club_rep':
             body['non_member_terms'] = new_terms
         else:
@@ -106,5 +106,5 @@ class AddUserController(Controller):
         self.model.first_name = data.get('given_name', '')
         self.model.last_name = data.get('sn', '')
         self.model.program = data.get('program', '')
-        self.model.forwarding_address = data.get('mail_local_addresses', [''])[0]
+        self.model.forwarding_address = (data.get('mail_local_addresses') or [''])[0]
         self.app.run_in_main_loop(self._on_lookup_user_success)

ceo/tui/controllers/GetPositionsController.py

@@ -19,8 +19,8 @@ class GetPositionsController(Controller):
             positions = tui_utils.handle_sync_response(resp, self)
         except Controller.RequestFailed:
             return
-        for pos, username in positions.items():
-            self.model.positions[pos] = username
+        for pos, usernames in positions.items():
+            self.model.positions[pos] = ','.join(usernames)
 
         def target():
             self.view.flash_text.set_text('')

ceo/tui/controllers/RenewUserController.py

@@ -28,7 +28,7 @@ class RenewUserController(SyncRequestController):
 
     def _get_next_terms(self):
         try:
-            self.model.new_terms = term_utils.get_terms_for_renewal(
+            self.model.new_terms = term_utils.get_terms_for_renewal_for_user(
                 self.model.username,
                 self.model.num_terms,
                 self.model.membership_type == 'club_rep',

ceo/tui/controllers/SearchGroupController.py

@@ -0,0 +1,40 @@
+from ceo.utils import http_get
+from .Controller import Controller
+from .SyncRequestController import SyncRequestController
+from ceo.tui.views import SearchGroupResponseView, GetGroupResponseView
+
+
+# this is a little bit bad because it relies on zero coupling between
+# the GetGroupResponseView and the GetGroupController
+# coupling is also introduced between this controller and the
+# SearchGroupResponseView as it requires this class's callback
+class SearchGroupController(SyncRequestController):
+    def __init__(self, model, app):
+        super().__init__(model, app)
+
+    def get_resp(self):
+        if self.model.want_info:
+            return http_get(f'/api/groups/{self.model.name}')
+        else:
+            return http_get(f'/api/groups/search/{self.model.name}/{self.model.count}')
+
+    def get_response_view(self):
+        if self.model.want_info:
+            return GetGroupResponseView(self.model, self, self.app)
+        else:
+            return SearchGroupResponseView(self.model, self, self.app)
+
+    def group_info_callback(self, button, cn):
+        self.model.name = cn
+        self.model.want_info = True
+        self.request_in_progress = False
+        self.on_next_button_pressed(button)
+
+    def on_next_button_pressed(self, button):
+        try:
+            if not self.model.want_info:
+                self.model.name = self.get_username_from_view()
+            self.model.count = 10
+        except Controller.InvalidInput:
+            return
+        self.on_confirmation_button_pressed(button)

ceo/tui/controllers/SetPositionsController.py

@@ -17,7 +17,7 @@ class SetPositionsController(Controller):
         body = {}
         for pos, field in self.view.position_fields.items():
             if field.edit_text != '':
-                body[pos] = field.edit_text
+                body[pos] = field.edit_text.replace(' ', '').split(',')
         model = TransactionModel(
             UpdateMemberPositionsTransaction.operations,
             'POST', '/api/positions', json=body
@@ -37,8 +37,8 @@ class SetPositionsController(Controller):
             positions = tui_utils.handle_sync_response(resp, self)
         except Controller.RequestFailed:
             return
-        for pos, username in positions.items():
-            self.model.positions[pos] = username
+        for pos, usernames in positions.items():
+            self.model.positions[pos] = ','.join(usernames)
 
         def target():
             self.view.flash_text.set_text('')

ceo/tui/controllers/__init__.py

@@ -8,6 +8,7 @@ from .ResetPasswordController import ResetPasswordController
 from .ChangeLoginShellController import ChangeLoginShellController
 from .AddGroupController import AddGroupController
 from .GetGroupController import GetGroupController
+from .SearchGroupController import SearchGroupController
 from .AddMemberToGroupController import AddMemberToGroupController
 from .RemoveMemberFromGroupController import RemoveMemberFromGroupController
 from .CreateDatabaseController import CreateDatabaseController

ceo/tui/models/SearchGroupModel.py

@@ -0,0 +1,9 @@
+class SearchGroupModel:
+    name = 'SearchGroup'
+    title = 'Search groups'
+
+    def __init__(self):
+        self.name = ''
+        self.resp_json = None
+        self.count = 10
+        self.want_info = False

ceo/tui/models/WelcomeModel.py

@@ -5,6 +5,7 @@ from .ResetPasswordModel import ResetPasswordModel
 from .ChangeLoginShellModel import ChangeLoginShellModel
 from .AddGroupModel import AddGroupModel
 from .GetGroupModel import GetGroupModel
+from .SearchGroupModel import SearchGroupModel
 from .AddMemberToGroupModel import AddMemberToGroupModel
 from .RemoveMemberFromGroupModel import RemoveMemberFromGroupModel
 from .CreateDatabaseModel import CreateDatabaseModel
@@ -29,6 +30,7 @@ class WelcomeModel:
             'Groups': [
                 AddGroupModel,
                 GetGroupModel,
+                SearchGroupModel,
                 AddMemberToGroupModel,
                 RemoveMemberFromGroupModel,
             ],

ceo/tui/models/__init__.py

@@ -6,6 +6,7 @@ from .ResetPasswordModel import ResetPasswordModel
 from .ChangeLoginShellModel import ChangeLoginShellModel
 from .AddGroupModel import AddGroupModel
 from .GetGroupModel import GetGroupModel
+from .SearchGroupModel import SearchGroupModel
 from .AddMemberToGroupModel import AddMemberToGroupModel
 from .RemoveMemberFromGroupModel import RemoveMemberFromGroupModel
 from .CreateDatabaseModel import CreateDatabaseModel

ceo/tui/utils.py

@@ -25,6 +25,7 @@ def handle_sync_response(resp, controller):
     raise Controller.RequestFailed()
 
 
+# this can probably be simplified with getattr or something
 def get_mvc(app, name):
     if name == WelcomeModel.name:
         model = WelcomeModel()
@@ -58,6 +59,10 @@ def get_mvc(app, name):
         model = GetGroupModel()
         controller = GetGroupController(model, app)
         view = GetGroupView(model, controller, app)
+    elif name == SearchGroupModel.name:
+        model = SearchGroupModel()
+        controller = SearchGroupController(model, app)
+        view = SearchGroupView(model, controller, app)
     elif name == AddMemberToGroupModel.name:
         model = AddMemberToGroupModel()
         controller = AddMemberToGroupController(model, app)

ceo/tui/views/AddUserView.py

@@ -53,6 +53,10 @@ class AddUserView(ColumnView):
                 urwid.Text('Program:', align='right'),
                 self.program_edit
             ),
+            (
+                urwid.Text('Forwarding address:', align='right'),
+                self.forwarding_address_edit
+            ),
             (
                 urwid.Text('Number of terms:', align='right'),
                 self.num_terms_edit

ceo/tui/views/SearchGroupResponseView.py

@@ -0,0 +1,21 @@
+import urwid
+
+from .ColumnResponseView import ColumnResponseView
+from .utils import decorate_button
+
+
+class SearchGroupResponseView(ColumnResponseView):
+    def __init__(self, model, controller, app):
+        super().__init__(model, controller, app)
+        matches = self.model.resp_json.copy()
+
+        rows = [(urwid.Text(resp),
+                decorate_button(urwid.Button('more info', on_press=self.create_callback(resp))))
+                for resp in matches if resp != '']
+
+        self.set_rows(rows, on_next=self.controller.get_next_menu_callback('Welcome'))
+
+    def create_callback(self, cn):
+        def callback(button):
+            self.controller.group_info_callback(button, cn)
+        return callback

ceo/tui/views/SearchGroupView.py

@@ -0,0 +1,17 @@
+import urwid
+
+from .ColumnView import ColumnView
+
+
+class SearchGroupView(ColumnView):
+    def __init__(self, model, controller, app):
+        super().__init__(model, controller, app)
+        # used for query, consider combining Controller.user_name_from_view and Controller.get_group_name_from_view
+        self.username_edit = urwid.Edit()
+        rows = [
+            (
+                urwid.Text('Query:', align='right'),
+                self.username_edit
+            )
+        ]
+        self.set_rows(rows)

ceo/tui/views/__init__.py

@@ -16,6 +16,8 @@ from .AddGroupView import AddGroupView
 from .AddGroupConfirmationView import AddGroupConfirmationView
 from .GetGroupView import GetGroupView
 from .GetGroupResponseView import GetGroupResponseView
+from .SearchGroupView import SearchGroupView
+from .SearchGroupResponseView import SearchGroupResponseView
 from .AddMemberToGroupView import AddMemberToGroupView
 from .AddMemberToGroupConfirmationView import AddMemberToGroupConfirmationView
 from .RemoveMemberFromGroupView import RemoveMemberFromGroupView

ceo/utils.py

@@ -87,6 +87,9 @@ def space_colon_kv(pairs: List[Tuple[str, str]]) -> List[str]:
     maxlen = max(len(key) for key, val in pairs)
     for key, val in pairs:
         if key != '':
+            if not val:
+                lines.append(key + ':')
+                continue
             prefix = key + ': '
         else:
             # assume this is a continuation from the previous line
@@ -136,6 +139,8 @@ def user_dict_kv(d: Dict) -> List[Tuple[str]]:
         pairs.append(('non-member terms', ','.join(_terms)))
     if 'password' in d:
         pairs.append(('password', d['password']))
+    if 'groups' in d:
+        pairs.append(('groups', ','.join(d['groups'])))
     return pairs
 
 

ceo_common/interfaces/ICloudStackService.py

@@ -19,8 +19,7 @@ class ICloudStackService(Interface):
         The dict is mapping of usernames to account IDs.
         """
 
-    def delete_account(account_id: str):
+    def delete_account(username: str, account_id: str):
         """
         Delete the given CloudStack account.
-        Note that a CloudStack account ID must be given, not a username.
         """

ceo_common/interfaces/ILDAPService.py

@@ -33,6 +33,11 @@ class ILDAPService(Interface):
         A new UID and GID will be generated and returned in the new user.
         """
 
+    def get_groups_for_user(username: str) -> List[str]:
+        """
+        Get a list of the groups to which the user belongs.
+        """
+
     def remove_user(user: IUser):
         """Remove this user from the database."""
 

ceo_common/model/HTTPClient.py

@@ -6,7 +6,7 @@ from requests_gssapi import HTTPSPNEGOAuth
 from zope import component
 from zope.interface import implementer
 
-from ceo_common.interfaces import IConfig, IHTTPClient
+from ceo_common.interfaces import IConfig, IHTTPClient, IKerberosService
 
 
 @implementer(IHTTPClient)
@@ -40,10 +40,18 @@ class HTTPClient:
                 'opportunistic_auth': True,
                 'target_name': gssapi.Name('ceod/' + host),
             }
-            if flask.has_request_context() and 'client_token' in g:
+            if flask.has_request_context():
                 # This is reached when we are the server and the client has
                 # forwarded their credentials to us.
-                spnego_kwargs['creds'] = gssapi.Credentials(token=g.client_token)
+                token = None
+                if g.get('need_admin_creds', False):
+                    # Some Kerberos bindings in some programming languages can't
+                    # perform delegation, so use the admin creds here.
+                    token = component.getUtility(IKerberosService).get_admin_creds_token()
+                elif 'client_token' in g:
+                    token = g.client_token
+                if token is not None:
+                    spnego_kwargs['creds'] = gssapi.Credentials(token=token)
             elif delegate:
                 # This is reached when we are the client and we want to
                 # forward our credentials to the server.

ceo_common/model/Term.py

@@ -1,4 +1,5 @@
 import datetime
+from typing import List, Union
 
 import ceo_common.utils as utils
 
@@ -81,3 +82,39 @@ class Term:
         month = self.seasons.index(c) * 4 + 1
         day = 1
         return datetime.datetime(year, month, day)
+
+
+# Utility functions
+
+def get_terms_for_new_user(num_terms: int) -> List[str]:
+    current_term = Term.current()
+    terms = [current_term + i for i in range(num_terms)]
+    return list(map(str, terms))
+
+
+def get_terms_for_renewal(
+    existing_terms: Union[List[str], None],
+    num_terms: int,
+) -> List[str]:
+    """Calculates the terms for which a member or club rep should be renewed.
+
+    :param terms: The existing terms for the user being renewed. If the user
+                  is being renewed as a regular member, these should be the
+                  member terms. If they are being renewed as a club rep, these
+                  should be the non-member terms.
+                  This may be None if the user does not have any terms of the
+                  appropriate type (an empty list is also acceptable).
+    :param num_terms: The number of terms for which the user is being renewed.
+    """
+    max_term = None
+    current_term = Term.current()
+    if existing_terms:
+        max_term = max(map(Term, existing_terms))
+
+    if max_term is not None and max_term >= current_term:
+        next_term = max_term + 1
+    else:
+        next_term = current_term
+
+    terms = [next_term + i for i in range(num_terms)]
+    return list(map(str, terms))

ceo_common/utils.py

@@ -1,6 +1,52 @@
 import datetime
 
 
+class fuzzy_result:
+    def __init__(self, string, score):
+        self.string = string
+        self.score = score
+
+    # consider a score worse if the edit distance is larger
+    def __lt__(self, other):
+        return self.score > other.score
+
+    def __gt__(self, other):
+        return self.score < other.score
+
+    def __le__(self, other):
+        return self.score >= other.score
+
+    def __ge__(self, other):
+        return self.score <= other.score
+
+    def __eq__(self, other):
+        return self.score == other.score
+
+    def __ne__(self, other):
+        return self.score != other.score
+
+
+# compute levenshtein edit distance, adapted from rosetta code
+def fuzzy_match(s1, s2):
+    if len(s1) == 0:
+        return len(s2)
+    if len(s2) == 0:
+        return len(s1)
+    edits = [i for i in range(len(s2) + 1)]
+    for i in range(len(s1)):
+        corner = i
+        edits[0] = i + 1
+        for j in range(len(s2)):
+            upper = edits[j + 1]
+            if s1[i] == s2[j]:
+                edits[j + 1] = corner
+            else:
+                m = min(corner, upper, edits[j])
+                edits[j + 1] = m + 1
+            corner = upper
+    return edits[-1]
+
+
 def get_current_datetime() -> datetime.datetime:
     # We place this in a separate function so that we can mock it out
     # in our unit tests.

ceod/api/groups.py

@@ -1,9 +1,11 @@
 from flask import Blueprint, request
+from flask.json import jsonify
 from zope import component
 
 from .utils import authz_restrict_to_syscom, is_truthy, \
     create_streaming_response, development_only
 from ceo_common.interfaces import ILDAPService
+from ceo_common.utils import fuzzy_result, fuzzy_match
 from ceod.transactions.groups import (
     AddGroupTransaction,
     AddMemberToGroupTransaction,
@@ -11,6 +13,8 @@ from ceod.transactions.groups import (
     DeleteGroupTransaction,
 )
 
+from heapq import heappushpop, nlargest
+
 bp = Blueprint('groups', __name__)
 
 
@@ -32,6 +36,22 @@ def get_group(group_name):
     return group.to_dict()
 
 
+@bp.route('/search/<query>/<count>')
+def search_group(query, count):
+    query = str(query)
+    count = int(count)
+    ldap_srv = component.getUtility(ILDAPService)
+    clubs = ldap_srv.get_clubs()
+    scores = [fuzzy_result("", 99999) for _ in range(count)]
+    for club in clubs:
+        score = fuzzy_match(query, str(club.cn))
+        result = fuzzy_result(str(club.cn), score)
+        heappushpop(scores, result)
+
+    result = [score.string for score in nlargest(count, scores)]
+    return jsonify(result)
+
+
 @bp.route('/<group_name>/members/<username>', methods=['POST'])
 @authz_restrict_to_syscom
 def add_member_to_group(group_name, username):

ceod/api/members.py

@@ -1,13 +1,14 @@
-from flask import Blueprint, g, request
+from flask import Blueprint, request
 from flask.json import jsonify
 from zope import component
 
 from .utils import authz_restrict_to_staff, authz_restrict_to_syscom, \
-    user_is_in_group, requires_authentication_no_realm, \
+    user_is_in_group, requires_authentication_no_realm, requires_admin_creds, \
     create_streaming_response, development_only, is_truthy
 from ceo_common.errors import BadRequest, UserAlreadySubscribedError, UserNotSubscribedError
 from ceo_common.interfaces import ILDAPService, IConfig, IMailService
 from ceo_common.logger_factory import logger_factory
+from ceo_common.model.Term import get_terms_for_new_user, get_terms_for_renewal
 from ceod.transactions.members import (
     AddMemberTransaction,
     ModifyMemberTransaction,
@@ -20,20 +21,29 @@ logger = logger_factory(__name__)
 
 
 @bp.route('/', methods=['POST'], strict_slashes=False)
+@requires_admin_creds
 @authz_restrict_to_staff
 def create_user():
+    # We need to use the admin creds here because office members may not
+    # directly create new LDAP records.
+
     body = request.get_json(force=True)
     terms = body.get('terms')
     non_member_terms = body.get('non_member_terms')
     if (terms and non_member_terms) or not (terms or non_member_terms):
         raise BadRequest('Must specify either terms or non-member terms')
+    if type(terms) is int:
+        terms = get_terms_for_new_user(terms)
+    elif type(non_member_terms) is int:
+        non_member_terms = get_terms_for_new_user(non_member_terms)
     for attr in ['uid', 'cn', 'given_name', 'sn']:
         if not body.get(attr):
             raise BadRequest(f"Attribute '{attr}' is missing or empty")
 
-    # We need to use the admin creds here because office members may not
-    # directly create new LDAP records.
-    g.need_admin_creds = True
+    if terms:
+        logger.info(f"Creating member {body['uid']} for terms {terms}")
+    else:
+        logger.info(f"Creating club rep {body['uid']} for non-member terms {non_member_terms}")
 
     txn = AddMemberTransaction(
         uid=body['uid'],
@@ -61,7 +71,9 @@ def get_user(auth_user: str, username: str):
         get_forwarding_addresses = True
     ldap_srv = component.getUtility(ILDAPService)
     user = ldap_srv.get_user(username)
-    return user.to_dict(get_forwarding_addresses)
+    user_dict = user.to_dict(get_forwarding_addresses)
+    user_dict['groups'] = ldap_srv.get_groups_for_user(username)
+    return user_dict
 
 
 @bp.route('/<username>', methods=['PATCH'])
@@ -81,40 +93,48 @@ def patch_user(auth_user: str, username: str):
 
 
 @bp.route('/<username>/renew', methods=['POST'])
+@requires_admin_creds
 @authz_restrict_to_staff
 def renew_user(username: str):
+    # We need to use the admin creds here because office members should
+    # not be able to directly modify the shadowExpire field; this could
+    # prevent syscom members from logging into the machines.
+
     body = request.get_json(force=True)
     terms = body.get('terms')
     non_member_terms = body.get('non_member_terms')
     if (terms and non_member_terms) or not (terms or non_member_terms):
         raise BadRequest('Must specify either terms or non-member terms')
 
-    # We need to use the admin creds here because office members should
-    # not be able to directly modify the shadowExpire field; this could
-    # prevent syscom members from logging into the machines.
-    g.need_admin_creds = True
-
     ldap_srv = component.getUtility(ILDAPService)
     cfg = component.getUtility(IConfig)
     user = ldap_srv.get_user(username)
     member_list = cfg.get('mailman3_new_member_list')
 
+    if type(terms) is int:
+        terms = get_terms_for_renewal(user.terms, terms)
+    elif type(non_member_terms) is int:
+        non_member_terms = get_terms_for_renewal(user.non_member_terms, non_member_terms)
+
     def unexpire(user):
         if user.shadowExpire:
             user.set_expired(False)
             try:
                 user.subscribe_to_mailing_list(member_list)
+                logger.debug(f'Subscribed {user.uid} to {member_list}')
             except UserAlreadySubscribedError:
-                pass
+                logger.debug(f'{user.uid} is already subscribed to {member_list}')
 
-    if body.get('terms'):
-        user.add_terms(body['terms'])
+    if terms:
+        logger.info(f"Renewing member {username} for terms {terms}")
+        user.add_terms(terms)
         unexpire(user)
-        return {'terms_added': body['terms']}
-    elif body.get('non_member_terms'):
-        user.add_non_member_terms(body['non_member_terms'])
+        return {'terms_added': terms}
+    elif non_member_terms:
+        logger.info(f"Renewing club rep {username} for non-member terms {non_member_terms}")
+        user.add_non_member_terms(non_member_terms)
         unexpire(user)
-        return {'non_member_terms_added': body['non_member_terms']}
+        return {'non_member_terms_added': non_member_terms}
     else:
         raise BadRequest('Must specify either terms or non-member terms')
 
@@ -149,11 +169,13 @@ def expire_users():
 
     if not dry_run:
         for member in members:
+            logger.info(f'Expiring {member.uid}')
             member.set_expired(True)
             try:
                 member.unsubscribe_from_mailing_list(member_list)
+                logger.debug(f'Unsubscribed {member.uid} from {member_list}')
             except UserNotSubscribedError:
-                pass
+                logger.debug(f'{member.uid} is already unsubscribed from {member_list}')
 
     return jsonify([member.uid for member in members])
 

ceod/api/positions.py

@@ -2,6 +2,7 @@ from flask import Blueprint, request
 from zope import component
 
 from .utils import authz_restrict_to_syscom, create_streaming_response
+from ceo_common.errors import BadRequest
 from ceo_common.interfaces import ILDAPService, IConfig
 from ceod.transactions.members import UpdateMemberPositionsTransaction
 
@@ -15,7 +16,9 @@ def get_positions():
     positions = {}
     for user in ldap_srv.get_users_with_positions():
         for position in user.positions:
-            positions[position] = user.uid
+            if position not in positions:
+                positions[position] = []
+            positions[position].append(user.uid)
 
     return positions
 
@@ -29,23 +32,31 @@ def update_positions():
     required = cfg.get('positions_required')
     available = cfg.get('positions_available')
 
-    # remove falsy values
-    body = {
-        positions: username for positions, username in body.items()
-        if username
-    }
-
-    for position in body.keys():
+    # remove falsy values and parse multiple users in each position
+    # Example: "user1,user2, user3" -> ["user1","user2","user3"]
+    position_to_usernames = {}
+    for position, usernames in body.items():
+        if not usernames:
+            continue
+        if type(usernames) is list:
+            position_to_usernames[position] = usernames
+        elif type(usernames) is str:
+            position_to_usernames[position] = usernames.replace(' ', '').split(',')
+        else:
+            raise BadRequest('usernames must be a list or comma-separated string')
+
+    # check for duplicates (i.e. one username specified twice in the same list)
+    for usernames in position_to_usernames.values():
+        if len(usernames) != len(set(usernames)):
+            raise BadRequest('username may only be specified at most once for a position')
+
+    for position in position_to_usernames.keys():
         if position not in available:
-            return {
-                'error': f'unknown position: {position}'
-            }, 400
+            raise BadRequest(f'unknown position: {position}')
 
     for position in required:
-        if position not in body:
-            return {
-                'error': f'missing required position: {position}'
-            }, 400
+        if position not in position_to_usernames:
+            raise BadRequest(f'missing required position: {position}')
 
-    txn = UpdateMemberPositionsTransaction(body)
+    txn = UpdateMemberPositionsTransaction(position_to_usernames)
     return create_streaming_response(txn)

ceod/api/utils.py

@@ -1,12 +1,9 @@
 import functools
-import grp
 import json
-import os
-import pwd
 import traceback
 from typing import Callable, List
 
-from flask import current_app, stream_with_context
+from flask import current_app, g, stream_with_context
 from zope import component
 
 from .spnego import requires_authentication
@@ -40,9 +37,24 @@ def requires_authentication_no_realm(f: Callable) -> Callable:
     return wrapper
 
 
-def user_is_in_group(user: str, group: str) -> bool:
-    """Returns True if `user` is in `group`, False otherwise."""
-    return user in grp.getgrnam(group).gr_mem
+def requires_admin_creds(f: Callable) -> Callable:
+    """
+    Forces the next LDAP connection to use the admin Kerberos credentials.
+    This must be used BEFORE any of the authz decorators, since those
+    may require an LDAP connection, which will get cached for later use.
+    """
+    @functools.wraps(f)
+    def wrapper(*args, **kwargs):
+        g.need_admin_creds = True
+        return f(*args, **kwargs)
+    return wrapper
+
+
+def user_is_in_group(username: str, group_name: str) -> bool:
+    """Returns True if `username` is in `group_name`, False otherwise."""
+    ldap_srv = component.getUtility(ILDAPService)
+    group = ldap_srv.get_group(group_name)
+    return username in group.members
 
 
 def authz_restrict_to_groups(f: Callable, allowed_groups: List[str]) -> Callable:
@@ -51,8 +63,6 @@ def authz_restrict_to_groups(f: Callable, allowed_groups: List[str]) -> Callable
     specified groups.
     """
 
-    allowed_group_ids = [grp.getgrnam(g).gr_gid for g in allowed_groups]
-
     @requires_authentication_no_realm
     @functools.wraps(f)
     def wrapper(_username: str, *args, **kwargs):
@@ -62,15 +72,14 @@ def authz_restrict_to_groups(f: Callable, allowed_groups: List[str]) -> Callable
         if username.startswith('ceod/'):
             # ceod services are always allowed to make internal calls
             return f(*args, **kwargs)
-        for gid in os.getgrouplist(username, pwd.getpwnam(username).pw_gid):
-            if gid in allowed_group_ids:
+        ldap_srv = component.getUtility(ILDAPService)
+        for group_name in ldap_srv.get_groups_for_user(username):
+            if group_name in allowed_groups:
                 return f(*args, **kwargs)
         logger.debug(
             f"User '{username}' denied since they are not in one of {allowed_groups}"