Merge branch 'master' into feature-59

.drone.yml

@@ -5,7 +5,7 @@ name: default
 steps:
   # use the step name to mock out the gethostname() call in our tests
 - name: phosphoric-acid
-  image: python:3.7-buster
+  image: python:3.9-bullseye
   # unfortunately we have to do everything in one step because there's no
   # way to share system packages between steps
   commands:
@@ -25,12 +25,12 @@ steps:
 
 services:
   - name: auth1
-    image: debian:buster
+    image: debian:bullseye
     commands:
       - .drone/auth1-setup.sh
       - sleep infinity
   - name: coffee
-    image: debian:buster
+    image: debian:bullseye
     commands:
       - .drone/coffee-setup.sh
       - sleep infinity

.drone/auth1-setup.sh

@@ -28,7 +28,6 @@ killall slapd || true
 service nslcd stop || true
 rm -rf /etc/ldap/slapd.d
 rm /var/lib/ldap/*
-cp /usr/share/slapd/DB_CONFIG /var/lib/ldap/DB_CONFIG
 cp .drone/slapd.conf /etc/ldap/slapd.conf
 cp .drone/ldap.conf /etc/ldap/ldap.conf
 cp /usr/share/doc/sudo-ldap/schema.OpenLDAP /etc/ldap/schema/sudo.schema

.drone/coffee-setup.sh

@@ -11,9 +11,9 @@ add_fqdn_to_hosts $(get_ip_addr auth1) auth1
 apt install --no-install-recommends -y default-mysql-server postgresql
 
 # MYSQL
-service mysql stop
-sed -E -i 's/^(bind-address[[:space:]]+= 127.0.0.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf
-service mysql start
+service mariadb stop
+sed -E -i 's/^(bind-address[[:space:]]+= 127\.0\.0\.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf
+service mariadb start
 cat <<EOF | mysql
 CREATE USER IF NOT EXISTS 'mysql' IDENTIFIED BY 'mysql';
 GRANT ALL PRIVILEGES ON *.* TO 'mysql' WITH GRANT OPTION;
@@ -21,7 +21,7 @@ EOF
 
 # POSTGRESQL
 service postgresql stop
-POSTGRES_DIR=/etc/postgresql/11/main
+POSTGRES_DIR=/etc/postgresql/*/main
 cat <<EOF > $POSTGRES_DIR/pg_hba.conf
 # TYPE  DATABASE        USER            ADDRESS                 METHOD
 local   all             postgres                                peer

.drone/common.sh

@@ -75,6 +75,7 @@ auth_setup() {
     # LDAP
     apt install -y --no-install-recommends libnss-ldapd
     service nslcd stop || true
+    mkdir -p /etc/ldap
     cp .drone/ldap.conf /etc/ldap/ldap.conf
     grep -Eq '^map group member uniqueMember$' /etc/nslcd.conf || \
         echo 'map group member uniqueMember' >> /etc/nslcd.conf

PACKAGING.md

@@ -9,6 +9,8 @@ Make sure your GPG key is in /srv/debian/gpg on potassium-benzoate. See
 [here](https://wiki.csclub.uwaterloo.ca/Debian_Repository#Step_1:_Add_to_Uploaders)
 for instructions.
 
+Make sure you are in the `csc-mirror` group too.
+
 ## Creating the package
 Use Docker/Podman to avoid screwing up your main system.
 For example, to create a package for bullseye (replace `podman` with `docker` in all instances below if you're using Docker):
@@ -58,7 +60,7 @@ podman cp pyceo-packaging:/home/max/repos/pyceo.tar.gz .
 (Replace `/home/max/repos` by the directory in the container with the tarball.)
 Now upload the tarball to a CSC machine, e.g.
 ```
-scp pyceo.tar.gz mannitol:~/
+scp pyceo.tar.gz mannitol:~
 ```
 SSH into that machine and extract the tarball into a separate directory:
 ```

README.md

@@ -16,7 +16,7 @@ Docker containers instead, which are much easier to work with than the VM.
 
 First, make sure you create the virtualenv:
 ```sh
-docker run --rm -v "$PWD:$PWD:z" -w "$PWD" python:3.7-buster sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
+docker run --rm -v "$PWD:$PWD:z" -w "$PWD" python:3.9-bullseye sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
 ```
 Then bring up the containers:
 ```sh

VERSION.txt

@@ -1 +1 @@
-1.0.23
+1.0.24

ceo/tui/views/position_names.py

@@ -9,12 +9,4 @@ position_names = {
     'imapd': "IMAPD",
     'webmaster': "Web Master",
     'offsck': "Office Manager",
-    'ext-affairs-lead': "External Affairs Lead",
-    'marketing-lead': "Marketing Lead",
-    'design-lead': "Design Lead",
-    'events-lead': "Events Lead",
-    'reps-lead': "Reps Lead",
-    'mods-lead': "Mods Lead",
-    'photography-lead': "Photography Lead",
-    'other': "Other",
 }

ceo/utils.py

@@ -136,6 +136,8 @@ def user_dict_kv(d: Dict) -> List[Tuple[str]]:
         pairs.append(('non-member terms', ','.join(_terms)))
     if 'password' in d:
         pairs.append(('password', d['password']))
+    if 'groups' in d:
+        pairs.append(('groups', ','.join(d['groups'])))
     return pairs
 
 

ceo_common/model/HTTPClient.py

@@ -6,7 +6,7 @@ from requests_gssapi import HTTPSPNEGOAuth
 from zope import component
 from zope.interface import implementer
 
-from ceo_common.interfaces import IConfig, IHTTPClient
+from ceo_common.interfaces import IConfig, IHTTPClient, IKerberosService
 
 
 @implementer(IHTTPClient)
@@ -40,10 +40,18 @@ class HTTPClient:
                 'opportunistic_auth': True,
                 'target_name': gssapi.Name('ceod/' + host),
             }
-            if flask.has_request_context() and 'client_token' in g:
+            if flask.has_request_context():
                 # This is reached when we are the server and the client has
                 # forwarded their credentials to us.
-                spnego_kwargs['creds'] = gssapi.Credentials(token=g.client_token)
+                token = None
+                if g.get('need_admin_creds', False):
+                    # Some Kerberos bindings in some programming languages can't
+                    # perform delegation, so use the admin creds here.
+                    token = component.getUtility(IKerberosService).get_admin_creds_token()
+                elif 'client_token' in g:
+                    token = g.client_token
+                if token is not None:
+                    spnego_kwargs['creds'] = gssapi.Credentials(token=token)
             elif delegate:
                 # This is reached when we are the client and we want to
                 # forward our credentials to the server.

ceod/api/members.py

@@ -71,7 +71,9 @@ def get_user(auth_user: str, username: str):
         get_forwarding_addresses = True
     ldap_srv = component.getUtility(ILDAPService)
     user = ldap_srv.get_user(username)
-    return user.to_dict(get_forwarding_addresses)
+    user_dict = user.to_dict(get_forwarding_addresses)
+    user_dict['groups'] = ldap_srv.get_groups_for_user(username)
+    return user_dict
 
 
 @bp.route('/<username>', methods=['PATCH'])

ceod/model/LDAPService.py

@@ -103,7 +103,7 @@ class LDAPService:
         conn.search(self.ldap_groups_base,
                     f'(uniqueMember={self.uid_to_dn(username)})',
                     attributes=['cn'])
-        return [entry.cn.value for entry in conn.entries]
+        return sorted([entry.cn.value for entry in conn.entries])
 
     def get_display_info_for_users(self, usernames: List[str]) -> List[Dict[str, str]]:
         if not usernames:
@@ -316,12 +316,12 @@ class LDAPService:
         self,
         dry_run: bool = False,
         members: Union[List[str], None] = None,
-        uwldap_batch_size: int = 10,
+        uwldap_batch_size: int = 100,
     ):
         if members:
             filter = '(|' + ''.join([f'(uid={uid})' for uid in members]) + ')'
         else:
-            filter = '(objectClass=*)'
+            filter = '(objectClass=member)'
         conn = self._get_ldap_conn()
         conn.search(
             self.ldap_users_base, filter, attributes=['uid', 'program'])
@@ -336,12 +336,17 @@ class LDAPService:
             batch_uids = uids[i:i + uwldap_batch_size]
             batch_uw_programs = uwldap_srv.get_programs_for_users(batch_uids)
             uw_programs.extend(batch_uw_programs)
+        # uw_programs[i] will be None if the 'ou' attribute was not
+        # present in UWLDAP, or if no UWLDAP entry was found at all
+        for i, uw_program in enumerate(uw_programs):
+            if uw_program in (None, 'expired', 'orphaned'):
+                # If the UWLDAP record is orphaned, nonexistent, or missing
+                # data, assume that the member graduated
+                uw_programs[i] = 'Alumni'
         users_to_change = [
             (uids[i], csc_programs[i], uw_programs[i])
             for i in range(len(uids))
-            if csc_programs[i] != uw_programs[i] and (
-                uw_programs[i] not in (None, 'expired', 'orphaned')
-            )
+            if csc_programs[i] != uw_programs[i]
         ]
         if dry_run:
             return users_to_change

ceod/model/VHostManager.py

@@ -4,7 +4,7 @@ import os
 import re
 import shutil
 import subprocess
-from typing import List, Dict, Tuple
+from typing import List, Dict, Tuple, Union
 
 import jinja2
 from zope import component
@@ -53,6 +53,7 @@ class VHostManager:
         self.max_vhosts_per_account = cfg.get('cloud vhosts_max_vhosts_per_account')
         self.vhost_ip_min = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_min'))
         self.vhost_ip_max = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_max'))
+        self.reload_web_server_cmd = cfg.get('cloud vhosts_reload_web_server_cmd')
 
         self.acme_challenge_dir = cfg.get('cloud vhosts_acme_challenge_dir')
         self.acme_dir = '/root/.acme.sh'
@@ -82,12 +83,12 @@ class VHostManager:
         """Return a list of all vhost files for this user."""
         return glob.glob(os.path.join(self.vhost_dir, username + '_*'))
 
-    def _run(self, args: List[str]):
-        subprocess.run(args, check=True)
+    def _run(self, args: Union[List[str], str], **kwargs):
+        subprocess.run(args, check=True, **kwargs)
 
     def _reload_web_server(self):
         logger.debug('Reloading NGINX')
-        self._run(['systemctl', 'reload', 'nginx'])
+        self._run(self.reload_web_server_cmd, shell=True)
 
     def is_valid_domain(self, username: str, domain: str) -> bool:
         if VALID_DOMAIN_RE.match(domain) is None:
@@ -150,7 +151,7 @@ class VHostManager:
             self.acme_sh, '--install-cert', '-d', domain,
             '--key-file', key_path,
             '--fullchain-file', cert_path,
-            '--reloadcmd', 'systemctl reload nginx',
+            '--reloadcmd', self.reload_web_server_cmd,
         ])
 
     def _delete_cert(self, domain: str, cert_path: str, key_path: str):

debian/changelog

@@ -1,3 +1,16 @@
+ceo (1.0.24-bullseye1) bullseye; urgency=high
+
+  * Add support for using number in member terms renwewal API
+  * Sort group member listing by WatIAM ID
+  * Add more logging for Cloudstack
+  * Use LDAP instead of NSS
+  * Fix shadowExpire deserialization
+  * Fix email formatting bug in ClubWebHostingService
+  * Check if mail_local_addresses exists in UWLDAP entry
+  * Remove override_dh_systemd_start
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Sun, 23 Oct 2022 21:41:00 -0400
+
 ceo (1.0.23-bullseye1) bullseye; urgency=high
 
   * Fix some bugs in ClubWebHostingService.

debian/control

@@ -6,7 +6,8 @@ Standards-Version: 4.3.0
 Vcs-Git: https://git.csclub.uwaterloo.ca/public/pyceo.git
 Vcs-Browser: https://git.csclub.uwaterloo.ca/public/pyceo
 Uploaders: Max Erenberg <merenber@csclub.uwaterloo.ca>,
-    Raymond Li <raymo@csclub.uwaterloo.ca>
+    Raymond Li <raymo@csclub.uwaterloo.ca>,
+    Edwin <e42zhang@csclub.uwaterloo.ca>
 Build-Depends: debhelper (>= 12.1.1),
     python3-dev (>= 3.7),
     python3-venv (>= 3.7),

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.6"
 
 x-common: &common
-  image: python:3.7-buster
+  image: python:3.9-bullseye
   volumes:
     - .:$PWD:z
   environment:
@@ -14,7 +14,7 @@ x-common: &common
 services:
   auth1:
     <<: *common
-    image: debian:buster
+    image: debian:bullseye
     hostname: auth1
     command: auth1
 

docs/openapi.yaml

@@ -942,6 +942,11 @@ components:
           $ref: "#/components/schemas/NonMemberTerms"
         forwarding_addresses:
           $ref: "#/components/schemas/ForwardingAddresses"
+        groups:
+          type: array
+          description: Groups for which this user is a member of
+          items:
+            $ref: "#/components/schemas/GroupCN"
     UWLDAPUser:
       type: object
       properties:

etc/ceo.ini

@@ -19,9 +19,7 @@ port = 9987
 [positions]
 required = president,vice-president,sysadmin
 available = president,vice-president,treasurer,secretary,
-            sysadmin,cro,librarian,imapd,webmaster,offsck,
-            ext-affairs-lead,marketing-lead,design-lead,events-lead,
-            reps-lead,mods-lead,photography-lead,other
+            sysadmin,cro,librarian,imapd,webmaster,offsck
 
 [mysql]
 host = caffeine

etc/ceod.ini

@@ -64,9 +64,7 @@ exec = exec,exec-moderators
 [positions]
 required = president,vice-president,sysadmin
 available = president,vice-president,treasurer,secretary,
-            sysadmin,cro,librarian,imapd,webmaster,offsck,
-            ext-affairs-lead,marketing-lead,design-lead,events-lead,
-            reps-lead,mods-lead,photography-lead,other
+            sysadmin,cro,librarian,imapd,webmaster,offsck
 
 [mysql]
 # This is only used on the database_host.
@@ -99,6 +97,7 @@ members_domain = csclub.cloud
 k8s_members_domain = k8s.csclub.cloud
 ip_range_min = 172.19.134.10
 ip_range_max = 172.19.134.160
+reload_web_server_cmd = /root/bin/reload-nginx.sh
 
 [k8s]
 members_clusterrole = csc-members-default

tests/ceo/cli/test_members.py

@@ -26,8 +26,9 @@ def test_members_get(cli_setup, ldap_user):
         f"home directory:       {ldap_user.home_directory}\n"
         f"is a club:            {ldap_user.is_club()}\n"
         f"is a club rep:        {ldap_user.is_club_rep}\n"
-        "forwarding addresses: \n"
+        f"forwarding addresses: \n"
         f"member terms:         {','.join(ldap_user.terms)}\n"
+        f"groups:               \n"
     )
     assert result.exit_code == 0
     assert result.output == expected

tests/ceo_dev.ini

@@ -15,9 +15,7 @@ port = 9987
 [positions]
 required = president,vice-president,sysadmin
 available = president,vice-president,treasurer,secretary,
-            sysadmin,cro,librarian,imapd,webmaster,offsck,
-            ext-affairs-lead,marketing-lead,design-lead,events-lead,
-            reps-lead,mods-lead,photography-lead,other
+            sysadmin,cro,librarian,imapd,webmaster,offsck
 
 [mysql]
 host = coffee

tests/ceod/api/test_members.py

@@ -127,6 +127,7 @@ def test_api_get_user(cfg, client, create_user_result):
     del old_data['password']
 
     status, data = client.get(f'/api/members/{uid}')
+    del data['groups']
     assert status == 200
     assert data == old_data
 
@@ -262,6 +263,7 @@ def test_authz_check(client, create_user_result):
     del old_data['password']
     del old_data['forwarding_addresses']
     _, data = client.get(f'/api/members/{uid}', principal='regular1')
+    del data['groups']
     assert data == old_data
 
     # If we're syscom but we don't pass credentials, the request should fail

tests/ceod_dev.ini

@@ -60,9 +60,7 @@ exec = exec
 [positions]
 required = president,vice-president,sysadmin
 available = president,vice-president,treasurer,secretary,
-            sysadmin,cro,librarian,imapd,webmaster,offsck,
-            ext-affairs-lead,marketing-lead,design-lead,events-lead,
-            reps-lead,mods-lead,photography-lead,other
+            sysadmin,cro,librarian,imapd,webmaster,offsck
 
 [mysql]
 username = mysql
@@ -93,6 +91,7 @@ members_domain = csclub.cloud
 k8s_members_domain = k8s.csclub.cloud
 ip_range_min = 172.19.134.10
 ip_range_max = 172.19.134.160
+reload_web_server_cmd = systemctl reload nginx
 
 [k8s]
 members_clusterrole = csc-members-default

tests/ceod_test_local.ini

@@ -59,9 +59,7 @@ exec = exec
 [positions]
 required = president,vice-president,sysadmin
 available = president,vice-president,treasurer,secretary,
-            sysadmin,cro,librarian,imapd,webmaster,offsck,
-            ext-affairs-lead,marketing-lead,design-lead,events-lead,
-            reps-lead,mods-lead,photography-lead,other
+            sysadmin,cro,librarian,imapd,webmaster,offsck
 
 [mysql]
 username = mysql
@@ -92,6 +90,7 @@ members_domain = csclub.cloud
 k8s_members_domain = k8s.csclub.cloud
 ip_range_min = 172.19.134.10
 ip_range_max = 172.19.134.160
+reload_web_server_cmd = systemctl reload nginx
 
 [k8s]
 members_clusterrole = csc-members-default

tests/conftest.py

@@ -298,6 +298,17 @@ def uwldap_srv(cfg, ldap_conn):
     delete_subtree(conn, base_dn)
 
     conn.add(base_dn, 'organizationalUnit')
+    conn.add(
+        f'uid=ctdalek,{base_dn}',
+        ['inetLocalMailRecipient', 'inetOrgPerson', 'organizationalPerson', 'person'],
+        {
+            'mailLocalAddress': 'ctdalek@uwaterloo.internal',
+            'ou': 'Math',
+            'cn': 'Calum T. Dalek',
+            'sn': 'Dalek',
+            'givenName': 'Calum',
+        },
+    )
     _uwldap_srv = UWLDAPService()
     component.getGlobalSiteManager().registerUtility(_uwldap_srv, IUWLDAPService)
     yield _uwldap_srv