fix home links

Apache's ProxyPass directive doesn't seem to be passing the URI for
FCGI. There's probably a way to configure this, but it's easier to just
use HTTP instead.

.drone.yml

@@ -5,34 +5,34 @@ name: default
 steps:
   # use the step name to mock out the gethostname() call in our tests
 - name: phosphoric-acid
-  image: python:3.7-buster
+  image: python:3.9-slim-bullseye
   # unfortunately we have to do everything in one step because there's no
   # way to share system packages between steps
   commands:
     # install dependencies
-    - apt update && apt install -y libkrb5-dev libpq-dev python3-dev libaugeas0
+    - apt update
+    - apt install --no-install-recommends -y gcc libkrb5-dev libaugeas0
     - python3 -m venv venv
     - . venv/bin/activate
-    - pip install -r dev-requirements.txt
+    - venv/bin/pip install -r dev-requirements.txt -r requirements.txt
-    - pip install -r requirements.txt
 
     # lint
     - flake8
 
     # unit + integration tests
-    - .drone/phosphoric-acid-setup.sh
+    - bash -c ". .drone/phosphoric-acid-setup.sh && IMAGE__setup && CONTAINER__setup"
     - pytest -v
 
 services:
   - name: auth1
-    image: debian:buster
+    image: debian:bullseye-slim
     commands:
-      - .drone/auth1-setup.sh
+      - bash -c ". .drone/auth1-setup.sh && IMAGE__setup && CONTAINER__setup"
       - sleep infinity
   - name: coffee
-    image: debian:buster
+    image: debian:bullseye-slim
     commands:
-      - .drone/coffee-setup.sh
+      - bash -c ". .drone/coffee-setup.sh && IMAGE__setup && CONTAINER__setup"
       - sleep infinity
 
 trigger:

.drone/adldap_data.ldif

@@ -0,0 +1,17 @@
+dn: ou=ADLDAP,dc=csclub,dc=internal
+objectClass: organizationalUnit
+ou: ADLDAP
+
+dn: cn=alumni1,ou=ADLDAP,dc=csclub,dc=internal
+description: One, Alumni
+givenName: Alumni
+sn: *One
+cn: alumni1
+sAMAccountName: alumni1
+displayName: alumni1
+mail: alumni1@alumni.uwaterloo.internal
+objectClass: mockADUser
+objectClass: inetOrgPerson
+objectClass: organizationalPerson
+objectClass: person
+objectClass: top

.drone/auth1-setup.sh

@@ -4,103 +4,128 @@ set -ex
 
 . .drone/common.sh
 
-# set FQDN in /etc/hosts
-add_fqdn_to_hosts $(get_ip_addr $(hostname)) auth1
-
-if [ -n "$CI" ]; then
-    # I'm not sure why, but we also need to remove the hosts entry for the
-    # container's real hostname, otherwise slapd only looks for the principal
-    # ldap/<container hostname> (this is with the sasl-host option)
-    sed -E "/\\b$(hostname)\\b/d" /etc/hosts > /tmp/hosts
-    cat /tmp/hosts > /etc/hosts
-    rm /tmp/hosts
-fi
-
-apt install -y psmisc
-
 # If we don't do this then OpenLDAP uses a lot of RAM
 ulimit -n 1024
 
-# LDAP
+CONTAINER__fix_hosts() {
-apt install -y --no-install-recommends slapd ldap-utils libnss-ldapd sudo-ldap
+    add_fqdn_to_hosts $(get_ip_addr $(hostname)) auth1
-# `service slapd stop` doesn't seem to work
+    if [ -n "$CI" ]; then
-killall slapd || true
+        # I'm not sure why, but we also need to remove the hosts entry for the
-service nslcd stop || true
+        # container's real hostname, otherwise slapd only looks for the principal
-rm -rf /etc/ldap/slapd.d
+        # ldap/<container hostname> (this is with the sasl-host option)
-rm /var/lib/ldap/*
+        sed -E "/\\b$(hostname)\\b/d" /etc/hosts > /tmp/hosts
-cp /usr/share/slapd/DB_CONFIG /var/lib/ldap/DB_CONFIG
+        cat /tmp/hosts > /etc/hosts
-cp .drone/slapd.conf /etc/ldap/slapd.conf
+        rm /tmp/hosts
-cp .drone/ldap.conf /etc/ldap/ldap.conf
+    fi
-cp /usr/share/doc/sudo-ldap/schema.OpenLDAP /etc/ldap/schema/sudo.schema
+}
-cp .drone/rfc2307bis.schema /etc/ldap/schema/
-cp .drone/csc.schema /etc/ldap/schema/
-chown -R openldap:openldap /etc/ldap/schema/ /var/lib/ldap/ /etc/ldap/
-sleep 0.5 && service slapd start
-grep -Eq '^map group member uniqueMember$' /etc/nslcd.conf || \
-     echo 'map group member uniqueMember' >> /etc/nslcd.conf
-sed -E -i 's/^uri .*$/uri ldap:\/\/auth1.csclub.internal/' /etc/nslcd.conf
-sed -E -i 's/^base .*$/base dc=csclub,dc=internal/' /etc/nslcd.conf
-cp .drone/nsswitch.conf /etc/nsswitch.conf
-service nslcd start
-ldapadd -c -f .drone/data.ldif -Y EXTERNAL -H ldapi:///
-if [ -z "$CI" ]; then
-    ldapadd -c -f .drone/uwldap_data.ldif -Y EXTERNAL -H ldapi:/// || true
-    # setup ldapvi for convenience
-    apt install -y vim ldapvi
-    echo 'export EDITOR=vim' >> /root/.bashrc
-    echo 'alias ldapvi="ldapvi -Y EXTERNAL -h ldapi:///"' >> /root/.bashrc
-fi
 
-# KERBEROS
+IMAGE__setup_ldap() {
-apt install -y krb5-admin-server krb5-user libpam-krb5 libsasl2-modules-gssapi-mit sasl2-bin
+    # In the "slim" Docker images, /usr/share/doc/* is excluded by default
-service krb5-admin-server stop || true
+    echo 'path-include /usr/share/doc/sudo-ldap/schema.OpenLDAP' > /etc/dpkg/dpkg.cfg.d/zz-ceo
-service krb5-kdc stop || true
+    apt install -y --no-install-recommends slapd ldap-utils libnss-ldapd sudo-ldap
-service saslauthd stop || true
+    # `service slapd stop` doesn't seem to work
-cp .drone/krb5.conf /etc/krb5.conf
+    killall slapd || true
-cp .drone/kdc.conf /etc/krb5kdc.conf
+    service nslcd stop || true
-echo '*/admin *' > /etc/krb5kdc/kadm5.acl
+    rm -rf /etc/ldap/slapd.d
-rm -f /var/lib/krb5kdc/*
+    rm /var/lib/ldap/*
-echo -e 'krb5\nkrb5' | krb5_newrealm
+    cp .drone/slapd.conf /etc/ldap/slapd.conf
-service krb5-kdc start
+    cp .drone/ldap.conf /etc/ldap/ldap.conf
-service krb5-admin-server start
+    cp /usr/share/doc/sudo-ldap/schema.OpenLDAP /etc/ldap/schema/sudo.schema
-rm -f /etc/krb5.keytab
+    cp .drone/{rfc2307bis,csc,mock_ad}.schema /etc/ldap/schema/
-cat <<EOF | kadmin.local
+    chown -R openldap:openldap /etc/ldap/schema/ /var/lib/ldap/ /etc/ldap/
+    sleep 0.5 && service slapd start
+    ldapadd -c -f .drone/data.ldif -Y EXTERNAL -H ldapi:///
+    if [ -z "$CI" ]; then
+        ldapadd -c -f .drone/uwldap_data.ldif -Y EXTERNAL -H ldapi:/// || true
+        ldapadd -c -f .drone/adldap_data.ldif -Y EXTERNAL -H ldapi:/// || true
+        # setup ldapvi for convenience
+        apt install -y --no-install-recommends vim ldapvi
+        grep -q 'export EDITOR' /root/.bashrc || \
+            echo 'export EDITOR=vim' >> /root/.bashrc
+        grep -q 'alias ldapvi' /root/.bashrc || \
+            echo 'alias ldapvi="ldapvi -Y EXTERNAL -h ldapi:///"' >> /root/.bashrc
+    fi
+}
+
+IMAGE__setup_krb5() {
+    apt install -y krb5-admin-server krb5-user libpam-krb5 libsasl2-modules-gssapi-mit sasl2-bin
+    service krb5-admin-server stop || true
+    service krb5-kdc stop || true
+    service saslauthd stop || true
+    cp .drone/krb5.conf /etc/krb5.conf
+    cp .drone/kdc.conf /etc/krb5kdc.conf
+    echo '*/admin *' > /etc/krb5kdc/kadm5.acl
+    rm -f /var/lib/krb5kdc/*
+    echo -e 'krb5\nkrb5' | krb5_newrealm
+    service krb5-kdc start
+    service krb5-admin-server start
+    rm -f /etc/krb5.keytab
+    cat <<EOF | kadmin.local
 addpol -minlength 4 default
 addprinc -pw krb5 sysadmin/admin
-addprinc -pw krb5 ctdalek
+addprinc -randkey ceod/admin
-addprinc -pw krb5 exec1
-addprinc -pw krb5 regular1
-addprinc -pw krb5 office1
 addprinc -randkey host/auth1.csclub.internal
 addprinc -randkey ldap/auth1.csclub.internal
 ktadd host/auth1.csclub.internal
 ktadd ldap/auth1.csclub.internal
 EOF
-groupadd keytab || true
+    # Add all of the people defined in data.ldif
-chgrp keytab /etc/krb5.keytab
+    for princ in ctdalek exec1 regular1 office1; do
-chmod 640 /etc/krb5.keytab
+        echo "addprinc -pw krb5 $princ" | kadmin.local
-usermod -a -G keytab openldap
+    done
-usermod -a -G sasl openldap
+    groupadd keytab || true
-cat <<EOF > /usr/lib/sasl2/slapd.conf
+    chgrp keytab /etc/krb5.keytab
+    chmod 640 /etc/krb5.keytab
+    usermod -a -G keytab openldap
+    usermod -a -G sasl openldap
+    cat <<EOF > /usr/lib/sasl2/slapd.conf
 mech_list: plain login gssapi external
 pwcheck_method: saslauthd
 EOF
-sed -E -i 's/^START=.*$/START=yes/' /etc/default/saslauthd
+    sed -E -i 's/^START=.*$/START=yes/' /etc/default/saslauthd
-sed -E -i 's/^MECHANISMS=.*$/MECHANISMS="kerberos5"/' /etc/default/saslauthd
+    sed -E -i 's/^MECHANISMS=.*$/MECHANISMS="kerberos5"/' /etc/default/saslauthd
-service saslauthd start
+}
-while true; do
-    killall slapd
-    sleep 1
-    if service slapd start; then
-        break
-    fi
-done
 
-# sync with phosphoric-acid
+IMAGE__setup() {
-nc -l 0.0.0.0 9000 &
+    # slapd needs /etc/hosts to be setup properly
-if [ -z "$CI" ]; then
+    CONTAINER__fix_resolv_conf
-    # sync with coffee
+    CONTAINER__fix_hosts
-    nc -l 0.0.0.0 9001 &
+
-    # sync with mail
+    apt update
-    nc -l 0.0.0.0 9002 &
+    # for the 'killall' command
-fi
+    apt install -y psmisc
+
+    IMAGE__setup_ldap
+    IMAGE__setup_krb5
+    IMAGE__common_setup
+
+    service slapd stop || true
+    killall slapd || true
+    service krb5-admin-server stop || true
+    service krb5-kdc stop || true
+    service saslauthd stop || true
+}
+
+CONTAINER__setup() {
+    CONTAINER__fix_resolv_conf
+    CONTAINER__fix_hosts
+
+    local started_slapd=false
+    for i in {1..5}; do
+        if service slapd start; then
+            started_slapd=true
+            break
+        fi
+        sleep 1
+    done
+    if [ $started_slapd != "true" ]; then
+        echo "Failed to start slapd" >&2
+        return 1
+    fi
+    service krb5-admin-server start
+    service krb5-kdc start
+    service saslauthd start
+    service nslcd start
+    # Let other containers know that we're ready
+    nc -l -k 0.0.0.0 9000 &
+}

.drone/coffee-setup.sh

@@ -4,25 +4,28 @@ set -ex
 
 . .drone/common.sh
 
-# set FQDN in /etc/hosts
+CONTAINER__fix_hosts() {
-add_fqdn_to_hosts $(get_ip_addr $(hostname)) coffee
+    add_fqdn_to_hosts $(get_ip_addr $(hostname)) coffee
-add_fqdn_to_hosts $(get_ip_addr auth1) auth1
+    add_fqdn_to_hosts $(get_ip_addr auth1) auth1
+}
 
-apt install --no-install-recommends -y default-mysql-server postgresql
+IMAGE__setup() {
+    IMAGE__ceod_setup
+    apt install --no-install-recommends -y default-mysql-server postgresql
 
-# MYSQL
+    # MYSQL
-service mysql stop
+    service mariadb stop
-sed -E -i 's/^(bind-address[[:space:]]+= 127.0.0.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf
+    sed -E -i 's/^(bind-address[[:space:]]+= 127\.0\.0\.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf
-service mysql start
+    service mariadb start
-cat <<EOF | mysql
+    cat <<EOF | mysql
 CREATE USER IF NOT EXISTS 'mysql' IDENTIFIED BY 'mysql';
 GRANT ALL PRIVILEGES ON *.* TO 'mysql' WITH GRANT OPTION;
 EOF
 
-# POSTGRESQL
+    # POSTGRESQL
-service postgresql stop
+    service postgresql stop
-POSTGRES_DIR=/etc/postgresql/11/main
+    local POSTGRES_DIR=/etc/postgresql/*/main
-cat <<EOF > $POSTGRES_DIR/pg_hba.conf
+    cat <<EOF > $POSTGRES_DIR/pg_hba.conf
 # TYPE  DATABASE        USER            ADDRESS                 METHOD
 local   all             postgres                                peer
 host    all             postgres        localhost               md5
@@ -36,19 +39,29 @@ local   sameuser        all                                     peer
 host    sameuser        all             0.0.0.0/0               md5
 host    sameuser        all             ::/0                    md5
 EOF
-grep -Eq "^listen_addresses = '*'$" $POSTGRES_DIR/postgresql.conf || \
+    grep -Eq "^listen_addresses = '*'$" $POSTGRES_DIR/postgresql.conf || \
-     echo "listen_addresses = '*'" >> $POSTGRES_DIR/postgresql.conf
+         echo "listen_addresses = '*'" >> $POSTGRES_DIR/postgresql.conf
-service postgresql start
+    service postgresql start
-su -c "
+    su -c "
-cat <<EOF | psql
+    cat <<EOF | psql
 ALTER USER postgres WITH PASSWORD 'postgres';
 REVOKE ALL ON SCHEMA public FROM public;
 GRANT ALL ON SCHEMA public TO postgres;
 EOF" postgres
 
-if [ -z "$CI" ]; then
+    service mariadb stop || true
-    auth_setup coffee
+    service postgresql stop || true
-fi
+}
 
-# sync with phosphoric-acid
+CONTAINER__setup() {
-nc -l 0.0.0.0 9000 &
+    CONTAINER__fix_resolv_conf
+    CONTAINER__fix_hosts
+    CONTAINER__ceod_setup
+    if [ -z "$CI" ]; then
+        CONTAINER__auth_setup coffee
+    fi
+    service mariadb start
+    service postgresql start
+    # sync with phosphoric-acid
+    nc -l -k 0.0.0.0 9000 &
+}

.drone/common.sh

@@ -1,80 +1,14 @@
-# TODO: fix Drone
-chmod 1777 /tmp
-
-# don't resolve container names to *real* CSC machines
-sed -E 's/([[:alnum:]-]+\.)*uwaterloo\.ca//g' /etc/resolv.conf > /tmp/resolv.conf
-# remove empty 'search' lines, if we created them
-sed -E -i '/^search[[:space:]]*$/d' /tmp/resolv.conf
-# also remove the 'rotate' option, since this can cause the Docker DNS server
-# to be circumvented
-sed -E -i '/^options.*\brotate/d' /tmp/resolv.conf
-cp /tmp/resolv.conf /etc/resolv.conf
-rm /tmp/resolv.conf
-
-# normally systemd creates /run/ceod for us
-mkdir -p /run/ceod
-
-# mock out systemctl
-ln -sf /bin/true /usr/local/bin/systemctl
-# mock out acme.sh
-mkdir -p /root/.acme.sh
-ln -sf /bin/true /root/.acme.sh/acme.sh
-# mock out kubectl
-cp .drone/mock_kubectl /usr/local/bin/kubectl
-chmod +x /usr/local/bin/kubectl
-# add k8s authority certificate
-mkdir -p /etc/csc
-cp .drone/k8s-authority.crt /etc/csc/k8s-authority.crt
-# openssl is actually already present in the python Docker image,
-# so we don't need to mock it out
-
-# netcat is used for synchronization between the containers
 export DEBIAN_FRONTEND=noninteractive
-apt update
-apt install -y netcat-openbsd
-if [ "$(hostname)" != auth1 ]; then
-    # ceod uses Augeas, which is not installed by default in the Python
-    # Docker container
-    apt install -y libaugeas0
-fi
 
-get_ip_addr() {
+# The IMAGE__ functions should be called when building the image.
-    getent hosts $1 | cut -d' ' -f1
+# The CONTAINER__ functions should be called when running an instance of the
-}
+# image in a container.
-
-add_fqdn_to_hosts() {
-    ip_addr=$1
-    hostname=$2
-    sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
-    cp /tmp/hosts /etc/hosts
-    rm /tmp/hosts
-    echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts
-}
-
-sync_with() {
-    host=$1
-    port=9000
-    if [ $# -eq 2 ]; then
-        port=$2
-    fi
-    synced=false
-    # give it 20 minutes (can be slow if you're using e.g. NFS or Ceph)
-    for i in {1..240}; do
-        if nc -vz $host $port ; then
-            synced=true
-            break
-        fi
-        sleep 5
-    done
-    test $synced = true
-}
-
-auth_setup() {
-    hostname=$1
 
+IMAGE__auth_setup() {
     # LDAP
     apt install -y --no-install-recommends libnss-ldapd
     service nslcd stop || true
+    mkdir -p /etc/ldap
     cp .drone/ldap.conf /etc/ldap/ldap.conf
     grep -Eq '^map group member uniqueMember$' /etc/nslcd.conf || \
         echo 'map group member uniqueMember' >> /etc/nslcd.conf
@@ -85,28 +19,98 @@ auth_setup() {
     # KERBEROS
     apt install -y krb5-user libpam-krb5 libsasl2-modules-gssapi-mit
     cp .drone/krb5.conf /etc/krb5.conf
+}
 
-    if [ $hostname = phosphoric-acid ]; then
+IMAGE__common_setup() {
-        sync_port=9000
+    apt update
-    elif [ $hostname = coffee ]; then
+    # netcat is used for synchronization between the containers
-        sync_port=9001
+    apt install -y netcat-openbsd
-    else
+    IMAGE__auth_setup
-        sync_port=9002
+}
-    fi
-    sync_with auth1 $sync_port
 
+IMAGE__ceod_setup() {
+    IMAGE__common_setup
+    # ceod uses Augeas, which is not installed by default in the Python
+    # Docker container
+    apt install -y libaugeas0
+}
+
+CONTAINER__fix_resolv_conf() {
+    # don't resolve container names to *real* CSC machines
+    sed -E 's/([[:alnum:]-]+\.)*uwaterloo\.ca//g' /etc/resolv.conf > /tmp/resolv.conf
+    # remove empty 'search' lines, if we created them
+    sed -E -i '/^search[[:space:]]*$/d' /tmp/resolv.conf
+    # also remove the 'rotate' option, since this can cause the Docker DNS server
+    # to be circumvented
+    sed -E -i '/^options.*\brotate/d' /tmp/resolv.conf
+    # we can't replace /etc/resolv.conf using 'mv' because it's mounted into the container
+    cp /tmp/resolv.conf /etc/resolv.conf
+    rm /tmp/resolv.conf
+}
+
+CONTAINER__auth_setup() {
+    local hostname=$1
+    sync_with auth1
+    service nslcd start
     rm -f /etc/krb5.keytab
     cat <<EOF | kadmin -p sysadmin/admin -w krb5
 addprinc -randkey host/$hostname.csclub.internal
-ktadd host/$hostname.csclub.internal
 addprinc -randkey ceod/$hostname.csclub.internal
+ktadd host/$hostname.csclub.internal
 ktadd ceod/$hostname.csclub.internal
 EOF
-    if [ $hostname = phosphoric-acid ]; then
+}
-        cat <<EOF | kadmin -p sysadmin/admin -w krb5
+
-addprinc -randkey ceod/admin
+CONTAINER__ceod_setup() {
-ktadd ceod/admin
+    # normally systemd creates /run/ceod for us
-EOF
+    mkdir -p /run/ceod
-    fi
+
-    service nslcd start
+    # mock out systemctl
+    ln -sf /bin/true /usr/local/bin/systemctl
+    # mock out acme.sh
+    mkdir -p /root/.acme.sh
+    ln -sf /bin/true /root/.acme.sh/acme.sh
+    # mock out kubectl
+    cp .drone/mock_kubectl /usr/local/bin/kubectl
+    chmod +x /usr/local/bin/kubectl
+    # add k8s authority certificate
+    mkdir -p /etc/csc
+    cp .drone/k8s-authority.crt /etc/csc/k8s-authority.crt
+    # openssl is actually already present in the python Docker image,
+    # so we don't need to mock it out
+}
+
+# Common utility functions
+
+get_ip_addr() {
+    # There appears to be a bug in newer versions of Podman where using both
+    # --name and --hostname causes a container to have two identical DNS
+    # entries, which causes `getent hosts` to print two lines.
+    # So we use `head -n 1` to select just the first line.
+    getent hosts $1 | head -n 1 | cut -d' ' -f1
+}
+
+add_fqdn_to_hosts() {
+    local ip_addr=$1
+    local hostname=$2
+    sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
+    # we can't replace /etc/hosts using 'mv' because it's mounted into the container
+    cp /tmp/hosts /etc/hosts
+    rm /tmp/hosts
+    echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts
+}
+
+sync_with() {
+    local host=$1
+    local port=9000
+    local synced=false
+    # give it 20 minutes (can be slow if you're using e.g. NFS or Ceph)
+    for i in {1..240}; do
+        if nc -vz $host $port ; then
+            synced=true
+            break
+        fi
+        sleep 5
+    done
+    test $synced = true
 }

.drone/data.ldif

@@ -186,3 +186,28 @@ objectClass: group
 objectClass: posixGroup
 cn: office1
 gidNumber: 20004
+
+dn: uid=alumni1,ou=People,dc=csclub,dc=internal
+cn: Alumni One
+givenName: Alumni
+sn: One
+userPassword: {SASL}alumni1@CSCLUB.INTERNAL
+loginShell: /bin/bash
+homeDirectory: /users/alumni1
+uid: alumni1
+uidNumber: 20005
+gidNumber: 20005
+objectClass: top
+objectClass: account
+objectClass: posixAccount
+objectClass: shadowAccount
+objectClass: member
+program: Alumni
+term: w2024
+
+dn: cn=alumni1,ou=Group,dc=csclub,dc=internal
+objectClass: top
+objectClass: group
+objectClass: posixGroup
+cn: alumni1
+gidNumber: 20005

.drone/mail-setup.sh

@@ -4,20 +4,27 @@ set -ex
 
 . .drone/common.sh
 
-# set FQDN in /etc/hosts
+CONTAINER__fix_hosts() {
-add_fqdn_to_hosts $(get_ip_addr $(hostname)) mail
+    add_fqdn_to_hosts $(get_ip_addr $(hostname)) mail
-add_fqdn_to_hosts $(get_ip_addr auth1) auth1
+    add_fqdn_to_hosts $(get_ip_addr auth1) auth1
+}
 
-. venv/bin/activate
+IMAGE__setup() {
-python -m tests.MockMailmanServer &
+    IMAGE__ceod_setup
-python -m tests.MockSMTPServer &
+}
-python -m tests.MockCloudStackServer &
-python -m tests.MockHarborServer &
 
-auth_setup mail
+CONTAINER__setup() {
-
+    CONTAINER__fix_resolv_conf
-# for the VHostManager
+    CONTAINER__fix_hosts
-mkdir -p /run/ceod/member-vhosts
+    CONTAINER__ceod_setup
-
+    CONTAINER__auth_setup mail
-# sync with phosphoric-acid
+    # for the VHostManager
-nc -l 0.0.0.0 9000 &
+    mkdir -p /run/ceod/member-vhosts
+    # mock services
+    venv/bin/python -m tests.MockMailmanServer &
+    venv/bin/python -m tests.MockSMTPServer &
+    venv/bin/python -m tests.MockCloudStackServer &
+    venv/bin/python -m tests.MockHarborServer &
+    # sync with phosphoric-acid
+    nc -l -k 0.0.0.0 9000 &
+}

.drone/mock_ad.schema

@@ -0,0 +1,10 @@
+# Mock Active Directory Schema
+
+attributetype ( 1.3.6.1.4.1.70000.1.1.1 NAME 'sAMAccountName'
+    EQUALITY caseIgnoreMatch
+    SUBSTR caseIgnoreSubstringsMatch
+    SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{256} )
+
+objectclass ( 1.3.6.1.4.1.70000.1.2.1 NAME 'mockADUser'
+    SUP top AUXILIARY
+    MUST ( sAMAccountName ) )

.drone/phosphoric-acid-setup.sh

@@ -4,29 +4,44 @@ set -ex
 
 . .drone/common.sh
 
-# set FQDN in /etc/hosts
+CONTAINER__fix_hosts() {
-add_fqdn_to_hosts "$(get_ip_addr $(hostname))" phosphoric-acid
+    add_fqdn_to_hosts "$(get_ip_addr $(hostname))" phosphoric-acid
-add_fqdn_to_hosts "$(get_ip_addr auth1)" auth1
+    add_fqdn_to_hosts "$(get_ip_addr auth1)" auth1
-add_fqdn_to_hosts "$(get_ip_addr coffee)" coffee
+    add_fqdn_to_hosts "$(get_ip_addr coffee)" coffee
-# mail container doesn't run in CI
+    # mail container doesn't run in CI
-if [ -z "$CI" ]; then
+    if [ -z "$CI" ]; then
-    add_fqdn_to_hosts $(get_ip_addr mail) mail
+        add_fqdn_to_hosts $(get_ip_addr mail) mail
-fi
+    fi
+}
 
-auth_setup phosphoric-acid
+CONTAINER__setup_userdirs() {
+    # initialize the skel directory
+    shopt -s dotglob
+    mkdir -p /users/skel
+    cp /etc/skel/* /users/skel/
 
-# initialize the skel directory
+    # create directories for users
-shopt -s dotglob
+    for user in ctdalek regular1 exec1; do
-mkdir -p /users/skel
+        mkdir -p /users/$user
-cp /etc/skel/* /users/skel/
+        chown $user:$user /users/$user
+    done
+}
 
-# create directories for users
+IMAGE__setup() {
-for user in ctdalek regular1 exec1; do
+    IMAGE__ceod_setup
-    mkdir -p /users/$user
+    # git is required by the ClubWebHostingService
-    chown $user:$user /users/$user
+    apt install --no-install-recommends -y git
-done
+}
 
-sync_with coffee
+CONTAINER__setup() {
-if [ -z "$CI" ]; then
+    CONTAINER__fix_resolv_conf
-    sync_with mail
+    CONTAINER__fix_hosts
-fi
+    CONTAINER__ceod_setup
+    CONTAINER__auth_setup phosphoric-acid
+    CONTAINER__setup_userdirs
+    echo "ktadd ceod/admin" | kadmin -p sysadmin/admin -w krb5
+    sync_with coffee
+    if [ -z "$CI" ]; then
+        sync_with mail
+    fi
+}

.drone/slapd.conf

@@ -7,6 +7,7 @@ include         /etc/ldap/schema/rfc2307bis.schema
 include         /etc/ldap/schema/inetorgperson.schema
 include         /etc/ldap/schema/sudo.schema
 include         /etc/ldap/schema/csc.schema
+include         /etc/ldap/schema/mock_ad.schema
 include         /etc/ldap/schema/misc.schema
 
 pidfile         /var/run/slapd/slapd.pid
@@ -40,6 +41,11 @@ access to *
         by dn="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" manage
         by * break
 
+# hide most attributes for alumni in mock UWLDAP
+access to attrs=cn,sn,givenName,displayName,ou,mail
+        dn.regex="^uid=alumni[^,]+,ou=(Test)?UWLDAP,dc=csclub,dc=internal$"
+        by * none
+
 # systems committee get full access
 access to *
         by dn="cn=ceod,dc=csclub,dc=internal" write

.drone/uwldap_data.ldif

@@ -106,3 +106,18 @@ objectClass: person
 objectClass: top
 uid: exec3
 mail: exec3@uwaterloo.internal
+
+dn: uid=alumni1,ou=UWLDAP,dc=csclub,dc=internal
+displayName: Alumni One
+givenName: Alumni
+sn: One
+cn: Alumni One
+ou: MAT/Mathematics Computer Science
+mailLocalAddress: alumni1@uwaterloo.internal
+objectClass: inetLocalMailRecipient
+objectClass: inetOrgPerson
+objectClass: organizationalPerson
+objectClass: person
+objectClass: top
+uid: alumni1
+mail: alumni1@uwaterloo.internal

.githooks/pre-commit

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+docker run --rm -v "$PWD:$PWD:z" -w "$PWD" python:3.9-bullseye scripts/lint-docker.sh
+
+exit $?

.gitignore

@@ -1,6 +1,10 @@
 # If you update this file, please also update the extend-diff-ignore option
 # in debian/source/options.
 
+*.key
+*.gpg
+*.pgp
+
 __pycache__/
 /venv/
 /dist/

PACKAGING.md

@@ -9,11 +9,16 @@ Make sure your GPG key is in /srv/debian/gpg on potassium-benzoate. See
 [here](https://wiki.csclub.uwaterloo.ca/Debian_Repository#Step_1:_Add_to_Uploaders)
 for instructions.
 
+Make sure you are in the `csc-mirror` group too.
+
 ## Creating the package
 Use Docker/Podman to avoid screwing up your main system.
 For example, to create a package for bullseye (replace `podman` with `docker` in all instances below if you're using Docker):
 ```sh
-podman run -it --name pyceo-packaging -v "$PWD":"$PWD" -w "$PWD" debian:bullseye bash
+podman run -it --name pyceo-packaging -v "$PWD":"$PWD":z -w "$PWD" --security-opt="label=disable" debian:bookworm bash
+# if disconnected from shell, reconnect with:
+podman start pyceo-packaging
+podman exec -it pyceo-packaging bash
 ```
 **Important**: Make sure to use a container image for the same distribution which you're packaging.
 For example, if you're creating a package for bullseye, you should be using the debian:bullseye
@@ -24,22 +29,39 @@ Here are some of the prerequisites you'll need to build the deb files
 ```sh
 apt update
 apt install -y devscripts debhelper git-buildpackage vim
+apt install -y python3-dev python3-venv libkrb5-dev libpq-dev libaugeas0 scdoc  # dependencies for building ceo
 ```
 Make sure to also install all of the packages in the 'Build-Depends' section in debian/control.
 
-Update VERSION.txt to the next version, and do a git commit.
+Update VERSION.txt to the next version, and do a git commit (or `dpkg-source --commit`).
 
-Now run `dch -i` and edit the changelog.
+Now run `dch -i` and edit the changelog (update version, add your uploader name/email, add changes).
 
 Now you will build a signed package. Place your key ID after the `-k` argument, e.g.
 ```sh
-gbp buildpackage --git-upstream-branch=master -k8E5568ABB0CF96BC367806ED127923BE10DA48DC
+# (pre-requisite) if container doesn't have your gpg key
+## step 1: export from host/another computer with your keyring
+gpg --armor --output private.key --export-secret-key <your pgp key's id email>
+## step 2: import into build container
+gpg --import private.key
+## step 3: find your key's public key
+gpg --list-secret-keys  # get key id
+## step 4: trust ids (before building)
+gpg --edit <pub key id>
+gpg> trust # run when gpg editing prompt appears
+   > 5     # "ultimate" trust
+gpg> save  # gpg will report no changes were made, but trust of ids should be changed
+
+# alternatively, sign with `debsign` after creating unsigned package
+
+# build (signed) package
+gbp buildpackage --git-upstream-branch=master -k8E5568ABB0CF96BC367806ED127923BE10DA48DC --lintian-opts --no-lintian
 ```
 This will create a bunch of files (deb, dsc, tar.gz, etc.) in the parent directory.
 
 Now do another git commit (since you edited the changelog file).
 
-To clean the packages:
+To clean the packages (run this after uploading, ie. **do NOT run this if you just finished building**):
 ```sh
 rm ../*.{xz,gz,dsc,build,buildinfo,changes,deb}
 ```
@@ -47,28 +69,30 @@ rm ../*.{xz,gz,dsc,build,buildinfo,changes,deb}
 ## Uploading the package
 Inside the container, go up one directory, and create a tarball with all the package files:
 ```
-cd ..
+cd ..  # within the container, generated files are in the parent directory of your git repo
 tar zcvf pyceo.tar.gz *.{xz,gz,dsc,build,buildinfo,changes,deb}
 ```
 Outside of the container (i.e. on your personal machine), copy the tarball out of the
 container into your current directory, e.g.
 ```
 podman cp pyceo-packaging:/home/max/repos/pyceo.tar.gz .
+# or generally, if you're in the pyceo repo:
+podman cp pyceo-packaging:$(cd ../ && pwd)/pyceo.tar.gz .
 ```
 (Replace `/home/max/repos` by the directory in the container with the tarball.)
 Now upload the tarball to a CSC machine, e.g.
 ```
-scp pyceo.tar.gz mannitol:~/
+# on "HOST" machine
+scp pyceo.tar.gz mannitol:~
 ```
 SSH into that machine and extract the tarball into a separate directory:
 ```
 ssh mannitol
-mkdir pyceo-parent
+mkdir pyceo-parent && mv pyceo.tar.gz pyceo-parent/ && cd pyceo-parent
-mv pyceo.tar.gz pyceo-parent/
+rm -iv *.{xz,gz,dsc,build,buildinfo,changes,deb}
-cd pyceo-parent
 tar zxvf pyceo.tar.gz
 ```
-At this point, you will need a dupload.conf file. Ask someone on syscom for a copy.
+At this point, you will need a dupload.conf file. Ask someone on syscom for a copy. Place the dupload config at `~/.dupload.conf` (as per manpage).
 
 Now upload the package to potassium-benzoate:
 ```
@@ -78,7 +102,21 @@ dupload *.changes
 
 Now SSH into potassium-benzoate and run the following:
 ```
+# note: this is AUTOMATICALLY done (within 10-20 minutes by a cron job)
 sudo /srv/debian/bin/rrr-incoming
 ```
 
+To check if mirror has accepted the new package, visit: http://debian.csclub.uwaterloo.ca/dists/bookworm/
+
+<<<<<<< Updated upstream
+=======
+To update CEO:
+```
+# repeat this for all systems, starting from ceod servers
+sudo apt update
+# NOTE: be careful of changing configs!!
+sudo apt install --only-upgrade ceod
+````
+
+>>>>>>> Stashed changes
 There, that wasn't so bad...right? :')

README.md

@@ -10,17 +10,29 @@ overview of its architecture.
 The API documentation is available as a plain HTML file in [docs/redoc-static.html](docs/redoc-static.html).
 
 ## Development
-### Docker
+### Podman
 If you are not modifying code related to email or Mailman, then you may use
-Docker containers instead, which are much easier to work with than the VM.
+Podman containers instead, which are much easier to work with than the VM.
 
-First, make sure you create the virtualenv:
+If you are using Podman, make sure to set the `DOCKER_HOST` environment variable
+if you have not done so already:
+```bash
+# Add the following to e.g. your ~/.bashrc
+export DOCKER_HOST=unix://$XDG_RUNTIME_DIR/podman/podman.sock
+```
+The Podman socket also needs to be running:
+```bash
+# Enabled by default on Debian, but not on Fedora
+systemctl --user enable --now podman.socket
+```
+
+First, create the container images:
 ```sh
-docker run --rm -v "$PWD:$PWD:z" -w "$PWD" python:3.7-buster sh -c 'apt update && apt install -y libaugeas0 && python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt'
+scripts/build-all-images.sh
 ```
 Then bring up the containers:
 ```sh
-docker-compose up -d  # or without -d to run in the foreground
+docker-compose up -d
 ```
 This will create some containers with the bare minimum necessary for ceod to
 run, and start ceod on each of phosphoric-acid, mail, and coffee container.
@@ -183,12 +195,12 @@ replaced by coffee).
 To run ceod on a single host (as root, since the app needs to read the keytab):
 ```sh
 export FLASK_APP=ceod.api
-export FLASK_ENV=development
+export FLASK_DEBUG=true
 flask run -h 0.0.0.0 -p 9987
 ```
 
 Sometimes changes you make in the source code don't show up while Flask
-is running. Stop the flask app (Ctrl-C), run `clear_cache.sh`, then
+is running. Stop the flask app (Ctrl-C), run `scripts/clear_cache.sh`, then
 restart the app.
 
 ## Interacting with the application

VERSION.txt

@@ -1 +1 @@
-1.0.23
+1.0.31

ceo/__main__.py

@@ -1,5 +1,4 @@
 import os
-import socket
 import sys
 
 from zope import component
@@ -9,6 +8,7 @@ from .krb_check import krb_check
 from .tui.start import main as tui_main
 from ceo_common.interfaces import IConfig, IHTTPClient
 from ceo_common.model import Config, HTTPClient
+from ceo_common.utils import is_in_development
 
 
 def register_services():
@@ -19,8 +19,7 @@ def register_services():
     if 'CEO_CONFIG' in os.environ:
         config_file = os.environ['CEO_CONFIG']
     else:
-        # This is a hack to determine if we're in the dev env or not
+        if is_in_development():
-        if socket.getfqdn().endswith('.csclub.internal'):
             config_file = './tests/ceo_dev.ini'
         else:
             config_file = '/etc/csc/ceo.ini'

ceo/cli/cloud.py

@@ -28,6 +28,11 @@ def activate():
         'Congratulations! Your cloud account has been activated.',
         f'You may now login into https://cloud.{base_domain} with your CSC credentials.',
         "Make sure to enter 'Members' for the domain (no quotes).",
+        '',
+        'Please note that your cloud account will be PERMANENTLY DELETED when',
+        'your CSC membership expires, so make sure to purchase enough membership',
+        'terms in advance. You will receive a warning email one week before your',
+        'cloud account is deleted, so please make sure to check your Junk folder.',
     ]
     for line in lines:
         click.echo(line)

ceo/cli/groups.py

@@ -149,3 +149,15 @@ def delete(group_name):
     click.confirm(f"Are you sure you want to delete {group_name}?", abort=True)
     resp = http_delete(f'/api/groups/{group_name}')
     handle_stream_response(resp, DeleteGroupTransaction.operations)
+
+
+@groups.command(short_help='Search for groups')
+@click.argument('query')
+@click.option('--count', default=10, help='number of results to show')
+def search(query, count):
+    check_if_in_development()
+    resp = http_get(f'/api/groups/search/{query}/{count}')
+    result = handle_sync_response(resp)
+    for cn in result:
+        if cn != "":
+            click.echo(cn)

ceo/cli/members.py

@@ -3,6 +3,8 @@ from typing import Dict
 
 import click
 from zope import component
+from ceo_common.utils import validate_username
+
 
 from ..term_utils import get_terms_for_renewal_for_user
 from ..utils import http_post, http_get, http_patch, http_delete, \
@@ -37,6 +39,11 @@ def add(username, cn, given_name, sn, program, num_terms, clubrep, forwarding_ad
     cfg = component.getUtility(IConfig)
     uw_domain = cfg.get('uw_domain')
 
+    # Verify that the username is valid before requesting data from UWLDAP
+    username_validator = validate_username(username)
+    if not username_validator.is_valid:
+        return click.echo("The provided username is invalid")
+
     # Try to get info from UWLDAP
     resp = http_get('/api/uwldap/' + username)
     if resp.ok:

ceo/cli/positions.py

@@ -16,13 +16,22 @@ def positions():
 def get():
     resp = http_get('/api/positions')
     result = handle_sync_response(resp)
-    print_colon_kv(result.items())
+    print_colon_kv([
+        (position, ', '.join(usernames))
+        for position, usernames in result.items()
+    ])
 
 
 @positions.command(short_help='Update positions')
 def set(**kwargs):
-    body = {k.replace('_', '-'): v for k, v in kwargs.items()}
+    body = {
-    print_body = {k: v or '' for k, v in body.items()}
+        k.replace('_', '-'): v.replace(' ', '').split(',') if v else None
+        for k, v in kwargs.items()
+    }
+    print_body = {
+        k: ', '.join(v) if v else ''
+        for k, v in body.items()
+    }
     click.echo('The positions will be updated:')
     print_colon_kv(print_body.items())
     click.confirm('Do you want to continue?', abort=True)

ceo/cli/utils.py

@@ -1,10 +1,9 @@
-import socket
-
 from typing import List, Tuple, Dict
 
 import click
 import requests
 
+from ceo_common.utils import is_in_development
 from ..utils import space_colon_kv, generic_handle_stream_response
 from .CLIStreamResponseHandler import CLIStreamResponseHandler
 
@@ -53,6 +52,6 @@ def handle_sync_response(resp: requests.Response):
 
 def check_if_in_development() -> bool:
     """Aborts if we are not currently in the dev environment."""
-    if not socket.getfqdn().endswith('.csclub.internal'):
+    if not is_in_development():
         click.echo('This command may only be called during development.')
         raise Abort()

ceo/tui/controllers/Controller.py

@@ -1,6 +1,7 @@
 from abc import ABC
 
 import ceo.tui.utils as utils
+from ceo_common.utils import validate_username
 
 
 # NOTE: one controller can control multiple views,
@@ -52,8 +53,9 @@ class Controller(ABC):
     def get_username_from_view(self):
         username = self.view.username_edit.edit_text
         # TODO: share validation logic between CLI and TUI
-        if not username:
+        verification_res = validate_username(username)
-            self.view.popup('Username must not be empty')
+        if not verification_res.is_valid:
+            self.view.popup(verification_res.error_message)
             raise Controller.InvalidInput()
         return username
 

ceo/tui/controllers/GetPositionsController.py

@@ -19,8 +19,8 @@ class GetPositionsController(Controller):
             positions = tui_utils.handle_sync_response(resp, self)
         except Controller.RequestFailed:
             return
-        for pos, username in positions.items():
+        for pos, usernames in positions.items():
-            self.model.positions[pos] = username
+            self.model.positions[pos] = ','.join(usernames)
 
         def target():
             self.view.flash_text.set_text('')

ceo/tui/controllers/SearchGroupController.py

@@ -0,0 +1,40 @@
+from ceo.utils import http_get
+from .Controller import Controller
+from .SyncRequestController import SyncRequestController
+from ceo.tui.views import SearchGroupResponseView, GetGroupResponseView
+
+
+# this is a little bit bad because it relies on zero coupling between
+# the GetGroupResponseView and the GetGroupController
+# coupling is also introduced between this controller and the
+# SearchGroupResponseView as it requires this class's callback
+class SearchGroupController(SyncRequestController):
+    def __init__(self, model, app):
+        super().__init__(model, app)
+
+    def get_resp(self):
+        if self.model.want_info:
+            return http_get(f'/api/groups/{self.model.name}')
+        else:
+            return http_get(f'/api/groups/search/{self.model.name}/{self.model.count}')
+
+    def get_response_view(self):
+        if self.model.want_info:
+            return GetGroupResponseView(self.model, self, self.app)
+        else:
+            return SearchGroupResponseView(self.model, self, self.app)
+
+    def group_info_callback(self, button, cn):
+        self.model.name = cn
+        self.model.want_info = True
+        self.request_in_progress = False
+        self.on_next_button_pressed(button)
+
+    def on_next_button_pressed(self, button):
+        try:
+            if not self.model.want_info:
+                self.model.name = self.get_username_from_view()
+            self.model.count = 10
+        except Controller.InvalidInput:
+            return
+        self.on_confirmation_button_pressed(button)

ceo/tui/controllers/SetPositionsController.py

@@ -17,7 +17,7 @@ class SetPositionsController(Controller):
         body = {}
         for pos, field in self.view.position_fields.items():
             if field.edit_text != '':
-                body[pos] = field.edit_text
+                body[pos] = field.edit_text.replace(' ', '').split(',')
         model = TransactionModel(
             UpdateMemberPositionsTransaction.operations,
             'POST', '/api/positions', json=body
@@ -37,8 +37,8 @@ class SetPositionsController(Controller):
             positions = tui_utils.handle_sync_response(resp, self)
         except Controller.RequestFailed:
             return
-        for pos, username in positions.items():
+        for pos, usernames in positions.items():
-            self.model.positions[pos] = username
+            self.model.positions[pos] = ','.join(usernames)
 
         def target():
             self.view.flash_text.set_text('')

ceo/tui/controllers/__init__.py

@@ -8,6 +8,7 @@ from .ResetPasswordController import ResetPasswordController
 from .ChangeLoginShellController import ChangeLoginShellController
 from .AddGroupController import AddGroupController
 from .GetGroupController import GetGroupController
+from .SearchGroupController import SearchGroupController
 from .AddMemberToGroupController import AddMemberToGroupController
 from .RemoveMemberFromGroupController import RemoveMemberFromGroupController
 from .CreateDatabaseController import CreateDatabaseController

ceo/tui/models/SearchGroupModel.py

@@ -0,0 +1,9 @@
+class SearchGroupModel:
+    name = 'SearchGroup'
+    title = 'Search groups'
+
+    def __init__(self):
+        self.name = ''
+        self.resp_json = None
+        self.count = 10
+        self.want_info = False

ceo/tui/models/WelcomeModel.py

@@ -5,6 +5,7 @@ from .ResetPasswordModel import ResetPasswordModel
 from .ChangeLoginShellModel import ChangeLoginShellModel
 from .AddGroupModel import AddGroupModel
 from .GetGroupModel import GetGroupModel
+from .SearchGroupModel import SearchGroupModel
 from .AddMemberToGroupModel import AddMemberToGroupModel
 from .RemoveMemberFromGroupModel import RemoveMemberFromGroupModel
 from .CreateDatabaseModel import CreateDatabaseModel
@@ -29,6 +30,7 @@ class WelcomeModel:
             'Groups': [
                 AddGroupModel,
                 GetGroupModel,
+                SearchGroupModel,
                 AddMemberToGroupModel,
                 RemoveMemberFromGroupModel,
             ],

ceo/tui/models/__init__.py

@@ -6,6 +6,7 @@ from .ResetPasswordModel import ResetPasswordModel
 from .ChangeLoginShellModel import ChangeLoginShellModel
 from .AddGroupModel import AddGroupModel
 from .GetGroupModel import GetGroupModel
+from .SearchGroupModel import SearchGroupModel
 from .AddMemberToGroupModel import AddMemberToGroupModel
 from .RemoveMemberFromGroupModel import RemoveMemberFromGroupModel
 from .CreateDatabaseModel import CreateDatabaseModel

ceo/tui/utils.py

@@ -25,6 +25,7 @@ def handle_sync_response(resp, controller):
     raise Controller.RequestFailed()
 
 
+# this can probably be simplified with getattr or something
 def get_mvc(app, name):
     if name == WelcomeModel.name:
         model = WelcomeModel()
@@ -58,6 +59,10 @@ def get_mvc(app, name):
         model = GetGroupModel()
         controller = GetGroupController(model, app)
         view = GetGroupView(model, controller, app)
+    elif name == SearchGroupModel.name:
+        model = SearchGroupModel()
+        controller = SearchGroupController(model, app)
+        view = SearchGroupView(model, controller, app)
     elif name == AddMemberToGroupModel.name:
         model = AddMemberToGroupModel()
         controller = AddMemberToGroupController(model, app)

ceo/tui/views/SearchGroupResponseView.py

@@ -0,0 +1,21 @@
+import urwid
+
+from .ColumnResponseView import ColumnResponseView
+from .utils import decorate_button
+
+
+class SearchGroupResponseView(ColumnResponseView):
+    def __init__(self, model, controller, app):
+        super().__init__(model, controller, app)
+        matches = self.model.resp_json.copy()
+
+        rows = [(urwid.Text(resp),
+                decorate_button(urwid.Button('more info', on_press=self.create_callback(resp))))
+                for resp in matches if resp != '']
+
+        self.set_rows(rows, on_next=self.controller.get_next_menu_callback('Welcome'))
+
+    def create_callback(self, cn):
+        def callback(button):
+            self.controller.group_info_callback(button, cn)
+        return callback

ceo/tui/views/SearchGroupView.py

@@ -0,0 +1,17 @@
+import urwid
+
+from .ColumnView import ColumnView
+
+
+class SearchGroupView(ColumnView):
+    def __init__(self, model, controller, app):
+        super().__init__(model, controller, app)
+        # used for query, consider combining Controller.user_name_from_view and Controller.get_group_name_from_view
+        self.username_edit = urwid.Edit()
+        rows = [
+            (
+                urwid.Text('Query:', align='right'),
+                self.username_edit
+            )
+        ]
+        self.set_rows(rows)

ceo/tui/views/__init__.py

@@ -16,6 +16,8 @@ from .AddGroupView import AddGroupView
 from .AddGroupConfirmationView import AddGroupConfirmationView
 from .GetGroupView import GetGroupView
 from .GetGroupResponseView import GetGroupResponseView
+from .SearchGroupView import SearchGroupView
+from .SearchGroupResponseView import SearchGroupResponseView
 from .AddMemberToGroupView import AddMemberToGroupView
 from .AddMemberToGroupConfirmationView import AddMemberToGroupConfirmationView
 from .RemoveMemberFromGroupView import RemoveMemberFromGroupView

ceo/tui/views/position_names.py

@@ -5,8 +5,15 @@ position_names = {
     'secretary': "Secretary",
     'sysadmin': "Sysadmin",
     'cro': "Chief Returning Officer",
-    'librarian': "Librarian",
-    'imapd': "IMAPD",
     'webmaster': "Web Master",
     'offsck': "Office Manager",
+    'events-lead': "Events Lead",
+    'ext-affairs-lead': "External Affairs Lead",
+    'marketing-lead': "Marketing Lead",
+    'design-lead': "Design Lead",
+    'reps-lead': "Reps Lead",
+    'mods-lead': "Mods Lead",
+    'photography-lead': "Photography Lead",
+    'codey-bot-lead': 'Codey Bot Lead',
+    'other': 'Other',
 }

ceo/utils.py

@@ -87,6 +87,9 @@ def space_colon_kv(pairs: List[Tuple[str, str]]) -> List[str]:
     maxlen = max(len(key) for key, val in pairs)
     for key, val in pairs:
         if key != '':
+            if not val:
+                lines.append(key + ':')
+                continue
             prefix = key + ': '
         else:
             # assume this is a continuation from the previous line
@@ -136,6 +139,8 @@ def user_dict_kv(d: Dict) -> List[Tuple[str]]:
         pairs.append(('non-member terms', ','.join(_terms)))
     if 'password' in d:
         pairs.append(('password', d['password']))
+    if 'groups' in d:
+        pairs.append(('groups', ','.join(d['groups'])))
     return pairs
 
 

ceo_common/interfaces/IADLDAPService.py

@@ -0,0 +1,18 @@
+import typing
+from typing import Optional
+
+from zope.interface import Interface
+
+if typing.TYPE_CHECKING:
+    # FIXME: circular import caused by lifting in __init__.py
+    from ..model.ADLDAPRecord import ADLDAPRecord
+
+
+class IADLDAPService(Interface):
+    """Represents the AD LDAP database."""
+
+    def get_user(username: str) -> Optional['ADLDAPRecord']:
+        """
+        Return the LDAP record for the given user, or
+        None if no such record exists.
+        """

ceo_common/interfaces/IUWLDAPService.py

@@ -1,20 +1,23 @@
-from typing import List, Union
+import typing
+from typing import List, Optional
 
 from zope.interface import Interface
 
+if typing.TYPE_CHECKING:
+    # FIXME: circular import caused by lifting in __init__.py
+    from ..model.UWLDAPRecord import UWLDAPRecord
+
 
 class IUWLDAPService(Interface):
     """Represents the UW LDAP database."""
 
-    def get_user(username: str):
+    def get_user(username: str) -> Optional['UWLDAPRecord']:
         """
         Return the LDAP record for the given user, or
         None if no such record exists.
-
-        :rtype: Union[UWLDAPRecord, None]
         """
 
-    def get_programs_for_users(usernames: List[str]) -> List[Union[str, None]]:
+    def get_programs_for_users(usernames: List[str]) -> List[Optional[str]]:
         """
         Return the programs for the given users from UWLDAP.
         If no record or program is found for a user, their entry in

ceo_common/interfaces/__init__.py

@@ -1,3 +1,4 @@
+from .IADLDAPService import IADLDAPService
 from .ICloudResourceManager import ICloudResourceManager
 from .ICloudStackService import ICloudStackService
 from .IKerberosService import IKerberosService

ceo_common/model/ADLDAPRecord.py

@@ -0,0 +1,57 @@
+import ldap3
+
+from .UWLDAPRecord import UWLDAPRecord
+
+
+class ADLDAPRecord:
+    """Represents a record from the AD LDAP."""
+
+    # These are just the ones in which we're interested
+    ldap_attributes = [
+        'sAMAccountName',
+        'mail',
+        'description',
+        'sn',
+        'givenName',
+    ]
+
+    def __init__(
+        self,
+        sam_account_name: str,
+        mail: str,
+        description: str,
+        sn: str,
+        given_name: str,
+    ):
+        self.sam_account_name = sam_account_name
+        self.mail = mail
+        self.description = description
+        self.sn = sn
+        self.given_name = given_name
+
+    @staticmethod
+    def deserialize_from_ldap(entry: ldap3.Entry):
+        """
+        Deserializes a dict returned from LDAP into an
+        ADLDAPRecord.
+        """
+        return ADLDAPRecord(
+            sam_account_name=entry.sAMAccountName.value,
+            mail=entry.mail.value,
+            description=entry.description.value,
+            sn=entry.sn.value,
+            given_name=entry.givenName.value,
+        )
+
+    def to_uwldap_record(self) -> UWLDAPRecord:
+        """Converts this AD LDAP record into a UW LDAP record."""
+        return UWLDAPRecord(
+            uid=self.sam_account_name,
+            mail_local_addresses=[self.mail],
+            program=None,
+            # The description attribute has the format "Lastname, Firstname"
+            cn=' '.join(self.description.split(', ')[::-1]),
+            # Alumni's last names are prepended with an asterisk
+            sn=(self.sn[1:] if self.sn[0] == '*' else self.sn),
+            given_name=self.given_name
+        )

ceo_common/model/HTTPClient.py

@@ -6,7 +6,7 @@ from requests_gssapi import HTTPSPNEGOAuth
 from zope import component
 from zope.interface import implementer
 
-from ceo_common.interfaces import IConfig, IHTTPClient
+from ceo_common.interfaces import IConfig, IHTTPClient, IKerberosService
 
 
 @implementer(IHTTPClient)
@@ -40,10 +40,18 @@ class HTTPClient:
                 'opportunistic_auth': True,
                 'target_name': gssapi.Name('ceod/' + host),
             }
-            if flask.has_request_context() and 'client_token' in g:
+            if flask.has_request_context():
                 # This is reached when we are the server and the client has
                 # forwarded their credentials to us.
-                spnego_kwargs['creds'] = gssapi.Credentials(token=g.client_token)
+                token = None
+                if g.get('need_admin_creds', False):
+                    # Some Kerberos bindings in some programming languages can't
+                    # perform delegation, so use the admin creds here.
+                    token = component.getUtility(IKerberosService).get_admin_creds_token()
+                elif 'client_token' in g:
+                    token = g.client_token
+                if token is not None:
+                    spnego_kwargs['creds'] = gssapi.Credentials(token=token)
             elif delegate:
                 # This is reached when we are the client and we want to
                 # forward our credentials to the server.

ceo_common/model/__init__.py

@@ -1,4 +1,6 @@
+from .ADLDAPRecord import ADLDAPRecord
 from .Config import Config
 from .HTTPClient import HTTPClient
 from .RemoteMailmanService import RemoteMailmanService
 from .Term import Term
+from .UWLDAPRecord import UWLDAPRecord

ceo_common/utils.py

@@ -1,7 +1,78 @@
 import datetime
+import re
+from dataclasses import dataclass
+import socket
+
+# TODO: disallow underscores. Will break many tests with usernames that include _
+VALID_USERNAME_RE = re.compile(r"^[a-z][a-z0-9-_]+$")
+
+
+class fuzzy_result:
+    def __init__(self, string, score):
+        self.string = string
+        self.score = score
+
+    # consider a score worse if the edit distance is larger
+    def __lt__(self, other):
+        return self.score > other.score
+
+    def __gt__(self, other):
+        return self.score < other.score
+
+    def __le__(self, other):
+        return self.score >= other.score
+
+    def __ge__(self, other):
+        return self.score <= other.score
+
+    def __eq__(self, other):
+        return self.score == other.score
+
+    def __ne__(self, other):
+        return self.score != other.score
+
+
+# compute levenshtein edit distance, adapted from rosetta code
+def fuzzy_match(s1, s2):
+    if len(s1) == 0:
+        return len(s2)
+    if len(s2) == 0:
+        return len(s1)
+    edits = [i for i in range(len(s2) + 1)]
+    for i in range(len(s1)):
+        corner = i
+        edits[0] = i + 1
+        for j in range(len(s2)):
+            upper = edits[j + 1]
+            if s1[i] == s2[j]:
+                edits[j + 1] = corner
+            else:
+                m = min(corner, upper, edits[j])
+                edits[j + 1] = m + 1
+            corner = upper
+    return edits[-1]
 
 
 def get_current_datetime() -> datetime.datetime:
     # We place this in a separate function so that we can mock it out
     # in our unit tests.
     return datetime.datetime.now()
+
+
+@dataclass
+class UsernameValidationResult:
+    is_valid: bool
+    error_message: str = ''
+
+
+def validate_username(username: str) -> UsernameValidationResult:
+    if not username:
+        return UsernameValidationResult(False, 'Username must not be empty')
+    if not VALID_USERNAME_RE.fullmatch(username):
+        return UsernameValidationResult(False, 'Username is invalid')
+    return UsernameValidationResult(True)
+
+
+def is_in_development() -> bool:
+    """This is a hack to determine if we're in the dev env or not"""
+    return socket.getfqdn().endswith('.csclub.internal')

ceod/api/app_factory.py

@@ -9,13 +9,13 @@ from .error_handlers import register_error_handlers
 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
     IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \
     ICloudStackService, ICloudResourceManager, IKubernetesService, IVHostManager, \
-    IContainerRegistryService, IClubWebHostingService
+    IContainerRegistryService, IClubWebHostingService, IADLDAPService
 from ceo_common.model import Config, HTTPClient, RemoteMailmanService
 from ceod.api.spnego import init_spnego
 from ceod.model import KerberosService, LDAPService, FileService, \
     MailmanService, MailService, UWLDAPService, CloudStackService, \
     CloudResourceManager, KubernetesService, VHostManager, \
-    ContainerRegistryService, ClubWebHostingService
+    ContainerRegistryService, ClubWebHostingService, ADLDAPService
 from ceod.db import MySQLService, PostgreSQLService
 
 
@@ -36,6 +36,15 @@ def create_app(flask_config={}):
         from ceod.api import members
         app.register_blueprint(members.bp, url_prefix='/api/members')
 
+        from ceod.api import groups
+        app.register_blueprint(groups.bp, url_prefix='/api/groups')
+
+        from ceod.api import positions
+        app.register_blueprint(positions.bp, url_prefix='/api/positions')
+
+        from ceod.api import uwldap
+        app.register_blueprint(uwldap.bp, url_prefix='/api/uwldap')
+
     # Only offer mailman API if this host is running Mailman
     if hostname == cfg.get('ceod_mailman_host'):
         from ceod.api import mailman
@@ -53,15 +62,6 @@ def create_app(flask_config={}):
         from ceod.api import webhosting
         app.register_blueprint(webhosting.bp, url_prefix='/api/webhosting')
 
-    from ceod.api import groups
-    app.register_blueprint(groups.bp, url_prefix='/api/groups')
-
-    from ceod.api import positions
-    app.register_blueprint(positions.bp, url_prefix='/api/positions')
-
-    from ceod.api import uwldap
-    app.register_blueprint(uwldap.bp, url_prefix='/api/uwldap')
-
     register_error_handlers(app)
 
     @app.route('/ping')
@@ -74,7 +74,7 @@ def create_app(flask_config={}):
 
 def register_services(app):
     # Config
-    if app.config.get('ENV') == 'development' and 'CEOD_CONFIG' not in os.environ:
+    if app.config.get('DEBUG') and 'CEOD_CONFIG' not in os.environ:
         with importlib.resources.path('tests', 'ceod_dev.ini') as p:
             config_file = p.__fspath__()
     else:
@@ -112,9 +112,13 @@ def register_services(app):
     mail_srv = MailService()
     component.provideUtility(mail_srv, IMailService)
 
-    # UWLDAPService
+    # UWLDAPService, ADLDAPService
-    uwldap_srv = UWLDAPService()
+    if hostname == cfg.get('ceod_admin_host'):
-    component.provideUtility(uwldap_srv, IUWLDAPService)
+        uwldap_srv = UWLDAPService()
+        component.provideUtility(uwldap_srv, IUWLDAPService)
+
+        adldap_srv = ADLDAPService()
+        component.provideUtility(adldap_srv, IADLDAPService)
 
     # ClubWebHostingService
     if hostname == cfg.get('ceod_webhosting_host'):

ceod/api/groups.py

@@ -1,9 +1,12 @@
 from flask import Blueprint, request
+from flask.json import jsonify
 from zope import component
 
 from .utils import authz_restrict_to_syscom, is_truthy, \
-    create_streaming_response, development_only
+    create_streaming_response, development_only, requires_admin_creds, \
+    requires_authentication_no_realm, user_is_in_group
 from ceo_common.interfaces import ILDAPService
+from ceo_common.utils import fuzzy_result, fuzzy_match
 from ceod.transactions.groups import (
     AddGroupTransaction,
     AddMemberToGroupTransaction,
@@ -11,6 +14,8 @@ from ceod.transactions.groups import (
     DeleteGroupTransaction,
 )
 
+from heapq import heappushpop, nlargest
+
 bp = Blueprint('groups', __name__)
 
 
@@ -32,9 +37,42 @@ def get_group(group_name):
     return group.to_dict()
 
 
+@bp.route('/search/<query>/<count>')
+def search_group(query, count):
+    query = str(query)
+    count = int(count)
+    ldap_srv = component.getUtility(ILDAPService)
+    clubs = ldap_srv.get_clubs()
+    scores = [fuzzy_result("", 99999) for _ in range(count)]
+    for club in clubs:
+        score = fuzzy_match(query, str(club.cn))
+        result = fuzzy_result(str(club.cn), score)
+        heappushpop(scores, result)
+
+    result = [score.string for score in nlargest(count, scores)]
+    return jsonify(result)
+
+
+def may_add_user_to_group(auth_username: str, group_name: str) -> bool:
+    # (is syscom) OR (group is office AND client is offsck)
+    if user_is_in_group(auth_username, 'syscom'):
+        return True
+    if group_name == 'office':
+        ldap_srv = component.getUtility(ILDAPService)
+        auth_user = ldap_srv.get_user(auth_username)
+        if 'offsck' in auth_user.positions:
+            return True
+    return False
+
+
 @bp.route('/<group_name>/members/<username>', methods=['POST'])
-@authz_restrict_to_syscom
+@requires_admin_creds
-def add_member_to_group(group_name, username):
+@requires_authentication_no_realm
+def add_member_to_group(auth_username: str, group_name: str, username: str):
+    # Admin creds are required because slapd does not support access control
+    # rules which use the client's attributes
+    if not may_add_user_to_group(auth_username, group_name):
+        return {'error': "not authorized to add user to group"}, 403
     subscribe_to_lists = is_truthy(
         request.args.get('subscribe_to_lists', 'true')
     )
@@ -47,8 +85,13 @@ def add_member_to_group(group_name, username):
 
 
 @bp.route('/<group_name>/members/<username>', methods=['DELETE'])
-@authz_restrict_to_syscom
+@requires_admin_creds
-def remove_member_from_group(group_name, username):
+@requires_authentication_no_realm
+def remove_member_from_group(auth_username: str, group_name: str, username: str):
+    # Admin creds are required because slapd does not support access control
+    # rules which use the client's attributes
+    if not may_add_user_to_group(auth_username, group_name):
+        return {'error': "not authorized to add user to group"}, 403
     unsubscribe_from_lists = is_truthy(
         request.args.get('unsubscribe_from_lists', 'true')
     )

ceod/api/members.py

@@ -9,6 +9,7 @@ from ceo_common.errors import BadRequest, UserAlreadySubscribedError, UserNotSub
 from ceo_common.interfaces import ILDAPService, IConfig, IMailService
 from ceo_common.logger_factory import logger_factory
 from ceo_common.model.Term import get_terms_for_new_user, get_terms_for_renewal
+from ceo_common.utils import validate_username
 from ceod.transactions.members import (
     AddMemberTransaction,
     ModifyMemberTransaction,
@@ -30,15 +31,22 @@ def create_user():
     body = request.get_json(force=True)
     terms = body.get('terms')
     non_member_terms = body.get('non_member_terms')
+
     if (terms and non_member_terms) or not (terms or non_member_terms):
         raise BadRequest('Must specify either terms or non-member terms')
     if type(terms) is int:
         terms = get_terms_for_new_user(terms)
     elif type(non_member_terms) is int:
         non_member_terms = get_terms_for_new_user(non_member_terms)
-    for attr in ['uid', 'cn', 'given_name', 'sn']:
+    for attr in ['uid', 'cn', 'given_name', 'sn', 'forwarding_addresses']:
         if not body.get(attr):
             raise BadRequest(f"Attribute '{attr}' is missing or empty")
+    if type(body['forwarding_addresses']) is not list:
+        raise BadRequest('forwarding_addresses must be a list of email addresses')
+
+    uid_validator = validate_username(body['uid'])
+    if not uid_validator.is_valid:
+        raise BadRequest("Attribute 'uid' is missing or invalid")
 
     if terms:
         logger.info(f"Creating member {body['uid']} for terms {terms}")
@@ -53,7 +61,7 @@ def create_user():
         program=body.get('program'),
         terms=terms,
         non_member_terms=non_member_terms,
-        forwarding_addresses=body.get('forwarding_addresses'),
+        forwarding_addresses=body['forwarding_addresses'],
     )
     return create_streaming_response(txn)
 
@@ -71,7 +79,9 @@ def get_user(auth_user: str, username: str):
         get_forwarding_addresses = True
     ldap_srv = component.getUtility(ILDAPService)
     user = ldap_srv.get_user(username)
-    return user.to_dict(get_forwarding_addresses)
+    user_dict = user.to_dict(get_forwarding_addresses)
+    user_dict['groups'] = ldap_srv.get_groups_for_user(username)
+    return user_dict
 
 
 @bp.route('/<username>', methods=['PATCH'])
@@ -119,9 +129,9 @@ def renew_user(username: str):
             user.set_expired(False)
             try:
                 user.subscribe_to_mailing_list(member_list)
-                logger.debug(f'Unsubscribed {user.uid} from {member_list}')
+                logger.debug(f'Subscribed {user.uid} to {member_list}')
             except UserAlreadySubscribedError:
-                logger.debug(f'{user.uid} is already unsubscribed from {member_list}')
+                logger.debug(f'{user.uid} is already subscribed to {member_list}')
 
     if terms:
         logger.info(f"Renewing member {username} for terms {terms}")
@@ -148,9 +158,12 @@ def reset_user_password(username: str):
 
 
 @bp.route('/<username>', methods=['DELETE'])
+@requires_admin_creds
 @authz_restrict_to_syscom
 @development_only
 def delete_user(username: str):
+    # We use the admin creds for the integration tests for the web app, which
+    # uses the ceod/<host> key
     txn = DeleteMemberTransaction(username)
     return create_streaming_response(txn)
 

ceod/api/positions.py

@@ -2,6 +2,7 @@ from flask import Blueprint, request
 from zope import component
 
 from .utils import authz_restrict_to_syscom, create_streaming_response
+from ceo_common.errors import BadRequest
 from ceo_common.interfaces import ILDAPService, IConfig
 from ceod.transactions.members import UpdateMemberPositionsTransaction
 
@@ -15,7 +16,9 @@ def get_positions():
     positions = {}
     for user in ldap_srv.get_users_with_positions():
         for position in user.positions:
-            positions[position] = user.uid
+            if position not in positions:
+                positions[position] = []
+            positions[position].append(user.uid)
 
     return positions
 
@@ -29,23 +32,31 @@ def update_positions():
     required = cfg.get('positions_required')
     available = cfg.get('positions_available')
 
-    # remove falsy values
+    # remove falsy values and parse multiple users in each position
-    body = {
+    # Example: "user1,user2, user3" -> ["user1","user2","user3"]
-        positions: username for positions, username in body.items()
+    position_to_usernames = {}
-        if username
+    for position, usernames in body.items():
-    }
+        if not usernames:
+            continue
+        if type(usernames) is list:
+            position_to_usernames[position] = usernames
+        elif type(usernames) is str:
+            position_to_usernames[position] = usernames.replace(' ', '').split(',')
+        else:
+            raise BadRequest('usernames must be a list or comma-separated string')
 
-    for position in body.keys():
+    # check for duplicates (i.e. one username specified twice in the same list)
+    for usernames in position_to_usernames.values():
+        if len(usernames) != len(set(usernames)):
+            raise BadRequest('username may only be specified at most once for a position')
+
+    for position in position_to_usernames.keys():
         if position not in available:
-            return {
+            raise BadRequest(f'unknown position: {position}')
-                'error': f'unknown position: {position}'
-            }, 400
 
     for position in required:
-        if position not in body:
+        if position not in position_to_usernames:
-            return {
+            raise BadRequest(f'missing required position: {position}')
-                'error': f'missing required position: {position}'
-            }, 400
 
-    txn = UpdateMemberPositionsTransaction(body)
+    txn = UpdateMemberPositionsTransaction(position_to_usernames)
     return create_streaming_response(txn)

ceod/api/utils.py

@@ -51,7 +51,12 @@ def requires_admin_creds(f: Callable) -> Callable:
 
 
 def user_is_in_group(username: str, group_name: str) -> bool:
-    """Returns True if `username` is in `group_name`, False otherwise."""
+    """
+    Returns True if `username` is in `group_name` (or starts with "ceod/"),
+    False otherwise.
+    """
+    if username.startswith("ceod/"):
+        return True
     ldap_srv = component.getUtility(ILDAPService)
     group = ldap_srv.get_group(group_name)
     return username in group.members
@@ -139,7 +144,7 @@ def create_streaming_response(txn: AbstractTransaction):
 def development_only(f: Callable) -> Callable:
     @functools.wraps(f)
     def wrapper(*args, **kwargs):
-        if current_app.config.get('ENV') == 'development' or \
+        if current_app.config.get('DEBUG') or \
                 current_app.config.get('TESTING'):
             return f(*args, **kwargs)
         return {

ceod/api/uwldap.py

@@ -3,15 +3,25 @@ from flask.json import jsonify
 from zope import component
 
 from .utils import authz_restrict_to_syscom, is_truthy
-from ceo_common.interfaces import IUWLDAPService, ILDAPService
+from ceo_common.interfaces import IUWLDAPService, IADLDAPService, ILDAPService
+from ceo_common.logger_factory import logger_factory
 
 bp = Blueprint('uwldap', __name__)
+logger = logger_factory(__name__)
 
 
 @bp.route('/<username>')
 def get_user(username: str):
     uwldap_srv = component.getUtility(IUWLDAPService)
     record = uwldap_srv.get_user(username)
+    if record is not None and not record.sn:
+        # Alumni are missing a lot of information in UWLDAP.
+        # Try AD LDAP instead
+        logger.debug('Querying %s from AD LDAP', username)
+        adldap_srv = component.getUtility(IADLDAPService)
+        ad_record = adldap_srv.get_user(username)
+        if ad_record is not None:
+            record = ad_record.to_uwldap_record()
     if record is None:
         return {
             'error': 'user not found',

ceod/model/ADLDAPService.py

@@ -0,0 +1,57 @@
+from typing import Optional
+
+import dns.resolver
+import ldap3
+from zope import component
+from zope.interface import implementer
+
+from ceo_common.interfaces import IADLDAPService, IConfig
+from ceo_common.logger_factory import logger_factory
+from ceo_common.model import ADLDAPRecord
+from ceo_common.utils import is_in_development
+
+logger = logger_factory(__name__)
+
+
+@implementer(IADLDAPService)
+class ADLDAPService:
+    def __init__(self):
+        cfg = component.getUtility(IConfig)
+        if is_in_development():
+            self.adldap_dns_srv_name = None
+            self.adldap_server_url = cfg.get('adldap_server_url')
+        else:
+            self.adldap_dns_srv_name = cfg.get('adldap_dns_srv_name')
+            # Perform the actual DNS query later so that we don't delay startup
+            self.adldap_server_url = None
+        self.adldap_base = cfg.get('adldap_base')
+
+    def _get_server_url(self) -> str:
+        assert self.adldap_dns_srv_name is not None
+        answers = dns.resolver.resolve(self.adldap_dns_srv_name, 'SRV')
+        target = answers[0].target.to_text()
+        # Strip the trailing '.'
+        target = target[:-1]
+        logger.debug('Using AD LDAP server %s', target)
+        # ldaps doesn't seem to work
+        return 'ldap://' + target
+
+    def _get_conn(self) -> ldap3.Connection:
+        if self.adldap_server_url is None:
+            self.adldap_server_url = self._get_server_url()
+        # When get_info=ldap3.SCHEMA (the default), ldap3 tries to search
+        # for schema information in 'CN=Schema,CN=Configuration,DC=ds,DC=uwaterloo,DC=ca',
+        # which doesn't exist so a LDAPNoSuchObjectResult exception is raised.
+        # To avoid this, we tell ldap3 not to look up any server info.
+        server = ldap3.Server(self.adldap_server_url, get_info=ldap3.NONE)
+        return ldap3.Connection(
+            server, auto_bind=True, read_only=True, raise_exceptions=True)
+
+    def get_user(self, username: str) -> Optional[ADLDAPRecord]:
+        conn = self._get_conn()
+        conn.search(
+            self.adldap_base, f'(sAMAccountName={username})',
+            attributes=ADLDAPRecord.ldap_attributes, size_limit=1)
+        if not conn.entries:
+            return None
+        return ADLDAPRecord.deserialize_from_ldap(conn.entries[0])

ceod/model/CloudStackService.py

@@ -62,6 +62,7 @@ class CloudStackService:
         domain_id = self._get_domain_id()
         url = self._create_url({
             'command': 'listAccounts',
+            'accounttype': '0',  # regular user (exclude domain admin)
             'domainid': domain_id,
             'details': 'min',
         })

ceod/model/LDAPService.py

@@ -103,7 +103,7 @@ class LDAPService:
         conn.search(self.ldap_groups_base,
                     f'(uniqueMember={self.uid_to_dn(username)})',
                     attributes=['cn'])
-        return [entry.cn.value for entry in conn.entries]
+        return sorted([entry.cn.value for entry in conn.entries])
 
     def get_display_info_for_users(self, usernames: List[str]) -> List[Dict[str, str]]:
         if not usernames:
@@ -112,14 +112,14 @@ class LDAPService:
         filter = '(|' + ''.join([f'(uid={uid})' for uid in usernames]) + ')'
         attributes = ['uid', 'cn', 'program']
         conn.search(self.ldap_users_base, filter, attributes=attributes)
-        return [
+        return sorted([
             {
                 'uid': entry.uid.value,
                 'cn': entry.cn.value,
                 'program': entry.program.value or 'Unknown',
             }
             for entry in conn.entries
-        ]
+        ], key=lambda member: member['uid'])
 
     def get_users_with_positions(self) -> List[IUser]:
         conn = self._get_ldap_conn()
@@ -316,12 +316,14 @@ class LDAPService:
         self,
         dry_run: bool = False,
         members: Union[List[str], None] = None,
+        # The UWLDAP server currently has a result set limit of 50
+        # Keep it low just to be safe
         uwldap_batch_size: int = 10,
     ):
         if members:
             filter = '(|' + ''.join([f'(uid={uid})' for uid in members]) + ')'
         else:
-            filter = '(objectClass=*)'
+            filter = '(objectClass=member)'
         conn = self._get_ldap_conn()
         conn.search(
             self.ldap_users_base, filter, attributes=['uid', 'program'])
@@ -336,12 +338,17 @@ class LDAPService:
             batch_uids = uids[i:i + uwldap_batch_size]
             batch_uw_programs = uwldap_srv.get_programs_for_users(batch_uids)
             uw_programs.extend(batch_uw_programs)
+        # uw_programs[i] will be None if the 'ou' attribute was not
+        # present in UWLDAP, or if no UWLDAP entry was found at all
+        for i, uw_program in enumerate(uw_programs):
+            if uw_program in (None, 'expired', 'orphaned'):
+                # If the UWLDAP record is orphaned, nonexistent, or missing
+                # data, assume that the member graduated
+                uw_programs[i] = 'Alumni'
         users_to_change = [
             (uids[i], csc_programs[i], uw_programs[i])
             for i in range(len(uids))
-            if csc_programs[i] != uw_programs[i] and (
+            if csc_programs[i] != uw_programs[i]
-                uw_programs[i] not in (None, 'expired', 'orphaned')
-            )
         ]
         if dry_run:
             return users_to_change
@@ -353,7 +360,11 @@ class LDAPService:
         return users_to_change
 
     def _get_club_uids(self, conn: ldap3.Connection) -> List[str]:
-        conn.search(self.ldap_users_base, '(objectClass=club)', attributes=['uid'])
+        conn.extend.standard.paged_search(self.ldap_users_base,
+                                          '(objectClass=club)',
+                                          attributes=['uid'],
+                                          paged_size=50,
+                                          generator=False)
         return [entry.uid.value for entry in conn.entries]
 
     def get_clubs(self) -> List[IGroup]:

ceod/model/UWLDAPService.py

@@ -1,11 +1,11 @@
-from typing import Union, List
+from typing import List, Optional
 
 import ldap3
 from zope import component
 from zope.interface import implementer
 
-from .UWLDAPRecord import UWLDAPRecord
 from ceo_common.interfaces import IUWLDAPService, IConfig
+from ceo_common.model import UWLDAPRecord
 
 
 @implementer(IUWLDAPService)
@@ -20,7 +20,7 @@ class UWLDAPService:
             self.uwldap_server_url, auto_bind=True, read_only=True,
             raise_exceptions=True)
 
-    def get_user(self, username: str) -> Union[UWLDAPRecord, None]:
+    def get_user(self, username: str) -> Optional[UWLDAPRecord]:
         conn = self._get_conn()
         conn.search(
             self.uwldap_base, f'(uid={username})',
@@ -29,7 +29,7 @@ class UWLDAPService:
             return None
         return UWLDAPRecord.deserialize_from_ldap(conn.entries[0])
 
-    def get_programs_for_users(self, usernames: List[str]) -> List[Union[str, None]]:
+    def get_programs_for_users(self, usernames: List[str]) -> List[Optional[str]]:
         filter_str = '(|' + ''.join([f'(uid={uid})' for uid in usernames]) + ')'
         programs = [None] * len(usernames)
         user_indices = {uid: i for i, uid in enumerate(usernames)}

ceod/model/VHostManager.py

@@ -4,7 +4,7 @@ import os
 import re
 import shutil
 import subprocess
-from typing import List, Dict, Tuple
+from typing import List, Dict, Tuple, Union
 
 import jinja2
 from zope import component
@@ -53,6 +53,7 @@ class VHostManager:
         self.max_vhosts_per_account = cfg.get('cloud vhosts_max_vhosts_per_account')
         self.vhost_ip_min = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_min'))
         self.vhost_ip_max = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_max'))
+        self.reload_web_server_cmd = cfg.get('cloud vhosts_reload_web_server_cmd')
 
         self.acme_challenge_dir = cfg.get('cloud vhosts_acme_challenge_dir')
         self.acme_dir = '/root/.acme.sh'
@@ -82,12 +83,12 @@ class VHostManager:
         """Return a list of all vhost files for this user."""
         return glob.glob(os.path.join(self.vhost_dir, username + '_*'))
 
-    def _run(self, args: List[str]):
+    def _run(self, args: Union[List[str], str], **kwargs):
-        subprocess.run(args, check=True)
+        subprocess.run(args, check=True, **kwargs)
 
     def _reload_web_server(self):
         logger.debug('Reloading NGINX')
-        self._run(['systemctl', 'reload', 'nginx'])
+        self._run(self.reload_web_server_cmd, shell=True)
 
     def is_valid_domain(self, username: str, domain: str) -> bool:
         if VALID_DOMAIN_RE.match(domain) is None:
@@ -150,7 +151,7 @@ class VHostManager:
             self.acme_sh, '--install-cert', '-d', domain,
             '--key-file', key_path,
             '--fullchain-file', cert_path,
-            '--reloadcmd', 'systemctl reload nginx',
+            '--reloadcmd', self.reload_web_server_cmd,
         ])
 
     def _delete_cert(self, domain: str, cert_path: str, key_path: str):

ceod/model/__init__.py

@@ -1,3 +1,4 @@
+from .ADLDAPService import ADLDAPService
 from .CloudResourceManager import CloudResourceManager
 from .CloudStackService import CloudStackService
 from .KerberosService import KerberosService
@@ -5,7 +6,6 @@ from .LDAPService import LDAPService, UserNotFoundError, GroupNotFoundError
 from .User import User
 from .Group import Group
 from .UWLDAPService import UWLDAPService
-from .UWLDAPRecord import UWLDAPRecord
 from .FileService import FileService
 from .SudoRole import SudoRole
 from .MailService import MailService

ceod/transactions/members/UpdateMemberPositionsTransaction.py

@@ -1,5 +1,5 @@
 from collections import defaultdict
-from typing import Dict
+from typing import Dict, List
 
 from zope import component
 
@@ -20,15 +20,19 @@ class UpdateMemberPositionsTransaction(AbstractTransaction):
         'subscribe_to_mailing_lists',
     ]
 
-    def __init__(self, positions_reversed: Dict[str, str]):
+    def __init__(self, position_to_usernames: Dict[str, List[str]]):
         # positions_reversed is position -> username
         super().__init__()
         self.ldap_srv = component.getUtility(ILDAPService)
 
         # Reverse the dict so it's easier to use (username -> positions)
         self.positions = defaultdict(list)
-        for position, username in positions_reversed.items():
+        for position, usernames in position_to_usernames.items():
-            self.positions[username].append(position)
+            if isinstance(usernames, list):
+                for username in usernames:
+                    self.positions[username].append(position)
+            else:
+                raise TypeError("Username(s) under each position must be a list")
 
         # a cached Dict of the Users who need to be modified (username -> User)
         self.users: Dict[str, IUser] = {}
@@ -42,7 +46,7 @@ class UpdateMemberPositionsTransaction(AbstractTransaction):
         mailing_lists = cfg.get('auxiliary mailing lists_exec')
 
         # position -> username
-        new_positions_reversed = {}  # For returning result
+        new_position_to_usernames = {}  # For returning result
 
         # retrieve User objects and cache them
         for username in self.positions:
@@ -64,7 +68,9 @@ class UpdateMemberPositionsTransaction(AbstractTransaction):
 
             self.old_positions[username] = old_positions
             for position in new_positions:
-                new_positions_reversed[position] = username
+                if position not in new_position_to_usernames:
+                    new_position_to_usernames[position] = []
+                new_position_to_usernames[position].append(username)
         yield 'update_positions_ldap'
 
         # update exec group in LDAP
@@ -97,7 +103,7 @@ class UpdateMemberPositionsTransaction(AbstractTransaction):
         else:
             yield 'subscribe_to_mailing_lists'
 
-        self.finish(new_positions_reversed)
+        self.finish(new_position_to_usernames)
 
     def rollback(self):
         if 'update_exec_group_ldap' in self.finished_operations:

debian/ceod.postinst

@@ -1,2 +0,0 @@
-#!/bin/sh
-chmod 600 /etc/csc/ceod.ini

debian/changelog

@@ -1,3 +1,65 @@
+ceo (1.0.31-bookworm1) bookworm; urgency=medium
+
+  * Allow office manager to manage office group
+  * Fix regression issue for new positions
+
+ -- Nathan <n4chung@csclub.uwaterloo.ca>  Wed, 21 Feb 2024 06:40:29 +0000
+
+ceo (1.0.30-bookworm1) bookworm; urgency=medium
+
+  * Support for new positions
+  * Improved username validation
+  * UWLDAP API integration for alumni information
+
+ -- Nathan <n4chung@csclub.uwaterloo.ca>  Sun, 04 Feb 2024 19:38:18 +0000
+
+ceo (1.0.29-bookworm1) bookworm; urgency=medium
+
+  * Upgrade dependencies
+  * Check that forwarding_addresses parameter is a list
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Tue, 01 Aug 2023 01:14:08 +0000
+
+ceo (1.0.28-bullseye1) bullseye; urgency=medium
+
+  * Upgrade dependencies
+  * Check that forwarding_addresses parameter is a list
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Tue, 01 Aug 2023 00:07:38 +0000
+
+ceo (1.0.27-bullseye1) bullseye; urgency=medium
+
+  * Make forwarding_addresses mandatory when creating new member
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Fri, 09 Jun 2023 06:43:10 +0000
+
+ceo (1.0.26-bullseye1.1) bullseye; urgency=high
+
+  * Reduce UWLDAP batch size from 100 to 10
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Mon, 13 Feb 2023 22:35:47 +0000
+
+ceo (1.0.25-bullseye1.1) bullseye; urgency=medium
+
+  * Support multiple users sharing the same position
+  * Show groups when retrieving user information
+  * Use admin Kerberos credentials when subscribing new member to csc-general
+
+ --  Max Erenberg <merenber@csclub.uwaterloo.ca>  Mon, 06 Feb 2023 05:01:46 +0000
+
+ceo (1.0.24-bullseye1) bullseye; urgency=high
+
+  * Add support for using number in member terms renwewal API
+  * Sort group member listing by WatIAM ID
+  * Add more logging for Cloudstack
+  * Use LDAP instead of NSS
+  * Fix shadowExpire deserialization
+  * Fix email formatting bug in ClubWebHostingService
+  * Check if mail_local_addresses exists in UWLDAP entry
+  * Remove override_dh_systemd_start
+
+ -- Max Erenberg <merenber@csclub.uwaterloo.ca>  Sun, 23 Oct 2022 21:41:00 -0400
+
 ceo (1.0.23-bullseye1) bullseye; urgency=high
 
   * Fix some bugs in ClubWebHostingService.

debian/compat

@@ -1 +1 @@
-10
+13

debian/control

@@ -2,27 +2,28 @@ Source: ceo
 Maintainer: Systems Committee <syscom@csclub.uwaterloo.ca>
 Section: admin
 Priority: optional
-Standards-Version: 4.3.0
+Standards-Version: 4.6.2
 Vcs-Git: https://git.csclub.uwaterloo.ca/public/pyceo.git
 Vcs-Browser: https://git.csclub.uwaterloo.ca/public/pyceo
 Uploaders: Max Erenberg <merenber@csclub.uwaterloo.ca>,
-    Raymond Li <raymo@csclub.uwaterloo.ca>
+    Raymond Li <raymo@csclub.uwaterloo.ca>,
-Build-Depends: debhelper (>= 12.1.1),
+    Nathan <n4chung@csclub.uwaterloo.ca>,
-    python3-dev (>= 3.7),
+    Edwin <e42zhang@csclub.uwaterloo.ca>
-    python3-venv (>= 3.7),
+Build-Depends: debhelper (>= 13),
-    libkrb5-dev (>= 1.17),
+    python3-dev (>= 3.9),
-    libpq-dev (>= 11.13),
+    python3-venv (>= 3.9),
-    libaugeas0 (>= 1.11),
+    libkrb5-dev (>= 1.18),
-    scdoc (>= 1.9)
+    libpq-dev (>= 13.9),
+    libaugeas0 (>= 1.12),
+    scdoc (>= 1.11)
 
 Package: ceo-common
 Architecture: amd64
-Depends: python3 (>= 3.7),
+Depends: python3 (>= 3.9),
-    krb5-user (>= 1.17),
+    krb5-user (>= 1.18),
-    libkrb5-3 (>= 1.17),
+    libkrb5-3 (>= 1.18),
-    libpq5 (>= 11.13),
+    libpq5 (>= 13.9),
-    libaugeas0 (>= 1.11),
+    libaugeas0 (>= 1.12),
-    ${python3:Depends},
     ${misc:Depends}
 Description: CSC Electronic Office common files
  This package contains the common files for the CSC Electronic Office.
@@ -40,6 +41,7 @@ Package: ceod
 Architecture: amd64
 Replaces: ceo-daemon
 Conflicts: ceo-daemon
+Pre-Depends: ${misc:Pre-Depends}
 Depends: ceo-common (= ${source:Version}), openssl (>= 1.1.1), ${misc:Depends}
 Description: CSC Electronic Office daemon
  This package contains the daemon for the CSC Electronic Office.

debian/rules

@@ -5,5 +5,14 @@
 
 override_dh_strip:
 
+override_dh_strip_nondeterminism:
+
 override_dh_shlibdeps:
 
+override_dh_makeshlibs:
+
+override_dh_dwz:
+
+override_dh_fixperms:
+	dh_fixperms
+	chmod 600 $(CURDIR)/debian/ceod/etc/csc/ceod.ini

dev-requirements.txt

@@ -1,6 +1,6 @@
-flake8==5.0.4
+flake8==6.1.0
-setuptools==65.4.1
+setuptools==68.0.0
-wheel==0.37.1
+wheel==0.41.0
-pytest==7.1.3
+pytest==7.4.0
 aiosmtpd==1.4.2
-aiohttp==3.8.3
+aiohttp==3.8.5

docker-compose.yml

@@ -1,39 +1,54 @@
 version: "3.6"
 
 x-common: &common
-  image: python:3.7-buster
   volumes:
-    - .:$PWD:z
+    - ./.drone:/app/.drone:ro
-  environment:
+    - ./docker-entrypoint.sh:/app/docker-entrypoint.sh:ro
-    FLASK_APP: ceod.api
+    - ceo-venv:/app/venv:ro
-    FLASK_ENV: development
+    - ./ceo:/app/ceo:ro
-  working_dir: $PWD
+    - ./ceo_common:/app/ceo_common:ro
+    - ./ceod:/app/ceod:ro
+    - ./tests:/app/tests:ro
+    # for flake8
+    - ./setup.cfg:/app/setup.cfg:ro
+    - ./web:/app/web:z
+  security_opt:
+    - label:disable
+  working_dir: /app
   entrypoint:
     - ./docker-entrypoint.sh
 
+x-ceod-common: &ceod-common
+  <<: *common
+  image: ceo-generic:bullseye
+  environment:
+    FLASK_APP: ceod.api
+    FLASK_DEBUG: "true"
+
 services:
   auth1:
     <<: *common
-    image: debian:buster
+    image: ceo-auth1:bullseye
     hostname: auth1
     command: auth1
 
   coffee:
-    <<: *common
+    <<: *ceod-common
+    image: ceo-coffee:bullseye
     command: coffee
     hostname: coffee
     depends_on:
       - auth1
 
   mail:
-    <<: *common
+    <<: *ceod-common
     command: mail
     hostname: mail
     depends_on:
       - auth1
 
   phosphoric-acid:
-    <<: *common
+    <<: *ceod-common
     command: phosphoric-acid
     hostname: phosphoric-acid
     depends_on:
@@ -41,4 +56,8 @@ services:
       - coffee
       - mail
 
+volumes:
+  ceo-venv:
+    external: true
+
 # vim: expandtab sw=2 ts=2

docker-entrypoint.sh

@@ -1,16 +1,21 @@
-#!/bin/sh -e
+#!/bin/bash
 
-if ! [ -d venv ]; then
+set -ex
-    echo "You need to create the virtualenv first!" >&2
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <host>" >&2
     exit 1
 fi
-
 host="$1"
-[ -x ".drone/$host-setup.sh" ] && ".drone/$host-setup.sh"
+if ! [ -d venv ]; then
+    echo "You need to mount the virtualenv" >&2
+    exit 1
+fi
+. .drone/$host-setup.sh
+CONTAINER__setup
 
 if [ "$host" = auth1 ]; then
     exec sleep infinity
 else
-    . venv/bin/activate
+    exec .drone/supervise.sh venv/bin/flask run -h 0.0.0.0 -p 9987
-    exec .drone/supervise.sh flask run -h 0.0.0.0 -p 9987
 fi

docs/openapi.yaml

@@ -61,9 +61,9 @@ paths:
                 program:
                   $ref: "#/components/schemas/Program"
                 terms:
-                  $ref: "#/components/schemas/Terms"
+                  $ref: "#/components/schemas/TermsOrNumTerms"
                 non_member_terms:
-                  $ref: "#/components/schemas/NonMemberTerms"
+                  $ref: "#/components/schemas/TermsOrNumTerms"
                 forwarding_addresses:
                   $ref: "#/components/schemas/ForwardingAddresses"
       responses:
@@ -161,17 +161,11 @@ paths:
               - type: object
                 properties:
                   terms:
-                    type: array
+                    $ref: "#/components/schemas/TermsOrNumTerms"
-                    description: Terms for which this user will be a member
-                    items:
-                      $ref: "#/components/schemas/Term"
               - type: object
                 properties:
                   non_member_terms:
-                    type: array
+                    $ref: "#/components/schemas/TermsOrNumTerms"
-                    description: Terms for which this user will be a club rep
-                    items:
-                      $ref: "#/components/schemas/Term"
             example: {"terms": ["f2021"]}
       responses:
         "200":
@@ -290,6 +284,26 @@ paths:
                       $ref: "#/components/schemas/UID"
         "404":
           $ref: "#/components/responses/GroupNotFoundErrorResponse"
+  /groups/{query}/{count}:
+    get: 
+      tags: ['groups']
+      summary: fuzzy search groups
+      description: >-
+        search count number of groups, returns a list of names sorted by levenshtein edit
+        distance.
+      parameters:
+      - name: query
+        in: path
+        description: query or string to search for
+        required: true
+        schema:
+          type: string
+      - name: count
+        in: path
+        description: number of results to return, returns empty strings if necessary
+        required: true
+        schema:
+          type: int
   /groups/{group_name}/members/{username}:
     post:
       tags: ['groups']
@@ -383,11 +397,14 @@ paths:
               schema:
                 type: object
                 additionalProperties:
-                  type: string
+                  type: array
+                  description: list of usernames
+                  items:
+                    type: string
                 example:
-                  president: user0
+                  president: ["user1"]
-                  vice-president: user1
+                  vice-president: ["user2", "user3"]
-                  sysadmin: user2
+                  sysadmin: ["user4"]
                   treasurer:
     post:
       tags: ['positions']
@@ -404,11 +421,18 @@ paths:
             schema:
               type: object
               additionalProperties:
-                type: string
+                oneOf:
+                  - type: string
+                    description: username or comma-separated list of usernames
+                  - type: array
+                    description: list of usernames
+                    items:
+                      type: string
               example:
-                president: user0
+                president: user1
-                vice-president: user1
+                vice-president: user2, user3
-                sysadmin: user2
+                secretary: ["user4", "user5"]
+                sysadmin: ["user6"]
                 treasurer:
       responses:
         "200":
@@ -422,7 +446,7 @@ paths:
                 {"status": "in progress", "operation": "update_positions_ldap"}
                 {"status": "in progress", "operation": "update_exec_group_ldap"}
                 {"status": "in progress", "operation": "subscribe_to_mailing_list"}
-                {"status": "completed", "result": "OK"}
+                {"status": "completed", "result": {"president": ["user1"],"vice-president": ["user2", "user3"],"secretary": ["user4". "user5"],"sysadmin": ["user6"]}}
         "400":
           description: Failed
           content:
@@ -883,14 +907,15 @@ components:
       example: MAT/Mathematics Computer Science
     Terms:
       type: array
-      description: Terms for which this user was a member
+      description: List of terms
-      items:
-        $ref: "#/components/schemas/Term"
-    NonMemberTerms:
-      type: array
-      description: Terms for which this user was a club rep
       items:
         $ref: "#/components/schemas/Term"
+    TermsOrNumTerms:
+      oneOf:
+        - type: integer
+          description: number of additional terms to add
+          example: 1
+        - $ref: "#/components/schemas/Terms"
     LoginShell:
       type: string
       description: Login shell
@@ -939,9 +964,14 @@ components:
         terms:
           $ref: "#/components/schemas/Terms"
         non_member_terms:
-          $ref: "#/components/schemas/NonMemberTerms"
+          $ref: "#/components/schemas/Terms"
         forwarding_addresses:
           $ref: "#/components/schemas/ForwardingAddresses"
+        groups:
+          type: array
+          description: Groups for which this user is a member of
+          items:
+            $ref: "#/components/schemas/GroupCN"
     UWLDAPUser:
       type: object
       properties:

etc/ceo.ini

@@ -18,8 +18,11 @@ port = 9987
 
 [positions]
 required = president,vice-president,sysadmin
-available = president,vice-president,treasurer,secretary,
+available = president,vice-president,sysadmin,treasurer,
-            sysadmin,cro,librarian,imapd,webmaster,offsck
+            secretary,cro,webmaster,offsck,ext-affairs-lead,
+            marketing-lead,design-lead,events-lead,reps-lead,
+            mods-lead,photography-lead,codey-bot-lead,other
+
 
 [mysql]
 host = caffeine

etc/ceod.ini

@@ -30,6 +30,10 @@ sudo_base = ou=SUDOers,dc=csclub,dc=uwaterloo,dc=ca
 server_url = ldaps://uwldap.uwaterloo.ca
 base = dc=uwaterloo,dc=ca
 
+[adldap]
+dns_srv_name = _ldap._tcp.teaching.ds.uwaterloo.ca
+base = dc=teaching,dc=ds,dc=uwaterloo,dc=ca
+
 [members]
 min_id = 20001
 max_id = 29999
@@ -63,8 +67,10 @@ exec = exec,exec-moderators
 
 [positions]
 required = president,vice-president,sysadmin
-available = president,vice-president,treasurer,secretary,
+available = president,vice-president,sysadmin,treasurer,
-            sysadmin,cro,librarian,imapd,webmaster,offsck
+            secretary,cro,webmaster,offsck,ext-affairs-lead,
+            marketing-lead,design-lead,events-lead,reps-lead,
+            mods-lead,photography-lead,codey-bot-lead,other
 
 [mysql]
 # This is only used on the database_host.
@@ -97,6 +103,7 @@ members_domain = csclub.cloud
 k8s_members_domain = k8s.csclub.cloud
 ip_range_min = 172.19.134.10
 ip_range_max = 172.19.134.160
+reload_web_server_cmd = /root/bin/reload-nginx.sh
 
 [k8s]
 members_clusterrole = csc-members-default

requirements.txt

@@ -1,16 +1,17 @@
-click==8.1.3
+click==8.1.6
-cryptography==35.0.0
+cryptography==41.0.2
-Flask==2.1.2
+dnspython==2.5.0
-gssapi==1.6.14
+Flask==2.3.2
-gunicorn==20.1.0
+gssapi==1.8.2
+gunicorn==21.2.0
 Jinja2==3.1.2
 ldap3==2.9.1
-mysql-connector-python==8.0.26
+mysql-connector-python==8.1.0
-psycopg2==2.9.1
+psycopg2-binary==2.9.6
 python-augeas==1.1.0
-requests==2.26.0
+requests==2.31.0
 requests-gssapi==1.2.3
 urwid==2.1.2
-Werkzeug==2.1.2
+Werkzeug==2.3.6
 zope.component==5.0.1
 zope.interface==5.4.0

scripts/build-all-images.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -eux
+
+SUITE=bullseye
+PYTHON_VER=3.9
+DOCKER=docker
+if command -v podman >/dev/null; then
+    DOCKER=podman
+    export BUILDAH_FORMAT=docker
+fi
+
+build_image() {
+    local HOST=$1
+    local BASE_IMAGE=$2
+    local IMAGE=ceo-$HOST:$SUITE
+    if $DOCKER image exists $IMAGE; then
+        return
+    fi
+    run=". .drone/$HOST-setup.sh && IMAGE__setup"
+    if [ $HOST = generic ]; then
+        run=". .drone/phosphoric-acid-setup.sh && IMAGE__setup"
+    fi
+    $DOCKER build -t $IMAGE -f - . <<EOF
+FROM $BASE_IMAGE
+WORKDIR /app
+COPY .drone .drone
+RUN [ "/bin/bash", "-c", "$run" ]
+ENTRYPOINT [ "./docker-entrypoint.sh" ]
+EOF
+}
+
+# Install the Python dependencies into a volume and re-use it for all the containers
+if ! $DOCKER volume exists ceo-venv; then
+    $DOCKER volume create ceo-venv
+    $DOCKER run \
+        --rm \
+        --security-opt label=disable \
+        -w /app/venv \
+        -v ceo-venv:/app/venv:z \
+        -v ./requirements.txt:/tmp/requirements.txt:ro \
+        -v ./dev-requirements.txt:/tmp/dev-requirements.txt:ro \
+        python:$PYTHON_VER-slim-$SUITE \
+        sh -c "apt update && apt install --no-install-recommends -y gcc libkrb5-dev libaugeas0 && python -m venv . && bin/pip install -r /tmp/requirements.txt -r /tmp/dev-requirements.txt"
+fi
+
+build_image auth1 debian:$SUITE-slim
+build_image coffee python:$PYTHON_VER-slim-$SUITE
+# Used by the phosphoric-acid and mail containers
+build_image generic python:$PYTHON_VER-slim-$SUITE

scripts/delete-all-images.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -eux
+
+DOCKER=docker
+if command -v podman >/dev/null; then
+    DOCKER=podman
+fi
+
+$DOCKER images --format="{{index .Names 0}}" | \
+    grep -E "^(localhost/)?ceo-" | \
+    while read name; do $DOCKER rmi $name; done
+
+if $DOCKER volume exists ceo-venv; then
+    $DOCKER volume rm ceo-venv
+fi

scripts/git-hook-install.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Install pre-configured git hooks
+git config --local core.hooksPath .githooks/

scripts/lint-docker.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. venv/bin/activate
+
+echo -e "\nLinting Python files with Flake8:\n"
+
+flake8
+PASS=$?
+
+echo -e "\nPython linting complete!\n"
+
+if [ "$PASS" -eq 0 ]; then
+  echo -e "\033[42mCOMMIT SUCCEEDED\033[0m\n"
+  exit $?
+else
+  echo -e "\033[41mCOMMIT FAILED:\033[0m Your commit contains files that should pass flake8 but do not. Please fix the flake8 errors and try again.\n"
+  exit 1
+fi

tests/ceo/cli/test_cloud.py

@@ -17,6 +17,11 @@ def test_cloud_account_activate(cli_setup, mock_cloud_server, new_user, cfg):
         'Congratulations! Your cloud account has been activated.\n'
         f'You may now login into https://cloud.{base_domain} with your CSC credentials.\n'
         "Make sure to enter 'Members' for the domain (no quotes).\n"
+        '\n'
+        'Please note that your cloud account will be PERMANENTLY DELETED when\n'
+        'your CSC membership expires, so make sure to purchase enough membership\n'
+        'terms in advance. You will receive a warning email one week before your\n'
+        'cloud account is deleted, so please make sure to check your Junk folder.\n'
     )
     assert result.exit_code == 0
     assert result.output == expected

tests/ceo/cli/test_db_mysql.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import time
 
 from click.testing import CliRunner
 from mysql.connector import connect
@@ -10,14 +11,24 @@ from ceo.cli import cli
 
 
 def mysql_attempt_connection(host, username, password):
+    time.sleep(0.5)
     with connect(
-            host=host,
+        host=host,
-            user=username,
+        user=username,
-            password=password,
+        password=password,
     ) as con, con.cursor() as cur:
-        cur.execute("SHOW DATABASES")
+        found = False
-        response = cur.fetchall()
+        # Sometimes, when running the tests locally, I've observed a race condition
-        assert len(response) == 2
+        # where another client can't "see" a database right after we create it.
+        # I only observed this after upgrading the containers to bullseye.
+        for _ in range(4):
+            cur.execute("SHOW DATABASES")
+            response = cur.fetchall()
+            if len(response) == 2:
+                found = True
+                break
+            time.sleep(0.25)
+        assert found
 
         with pytest.raises(ProgrammingError):
             cur.execute("CREATE DATABASE new_db")
@@ -34,7 +45,7 @@ def test_mysql(cli_setup, cfg, ldap_user):
 
     # create database for user
     result = runner.invoke(cli, ['mysql', 'create', username], input='y\n')
-    print(result.output)
+    #print(result.output)  # noqa: E265
     assert result.exit_code == 0
     assert os.path.isfile(info_file_path)
 

tests/ceo/cli/test_groups.py

@@ -77,6 +77,27 @@ def test_groups(cli_setup, ldap_user):
     result = runner.invoke(cli, ['groups', 'delete', 'test_group_1'], input='y\n')
     assert result.exit_code == 0
 
+    group_names = [
+        "touch",
+        "error",
+        "happy",
+    ]
+    runner = CliRunner()
+    for name in group_names:
+        result = runner.invoke(cli, [
+            'groups', 'add', name, '-d', 'searchable group',
+        ], input='y\n')
+        assert result.exit_code == 0
+
+    for name in group_names:
+        result = runner.invoke(cli, ['groups', 'search', '--count=1', name])
+        assert result.exit_code == 0
+        assert result.output.find(name) != -1
+
+    for name in group_names:
+        result = runner.invoke(cli, ['groups', 'delete', name], input='y\n')
+        assert result.exit_code == 0
+
 
 def create_group(group_name, desc):
     runner = CliRunner()

tests/ceo/cli/test_members.py

@@ -26,8 +26,9 @@ def test_members_get(cli_setup, ldap_user):
         f"home directory:       {ldap_user.home_directory}\n"
         f"is a club:            {ldap_user.is_club()}\n"
         f"is a club rep:        {ldap_user.is_club_rep}\n"
-        "forwarding addresses: \n"
+        f"forwarding addresses:\n"
         f"member terms:         {','.join(ldap_user.terms)}\n"
+        f"groups:\n"
     )
     assert result.exit_code == 0
     assert result.output == expected
@@ -135,12 +136,13 @@ def test_members_renew(cli_setup, ldap_user, g_admin_ctx):
     assert result.output == expected
 
 
-def test_members_pwreset(cli_setup, ldap_user, krb_user):
+def test_members_pwreset(cli_setup, ldap_and_krb_user):
+    uid = ldap_and_krb_user.uid
     runner = CliRunner()
     result = runner.invoke(
-        cli, ['members', 'pwreset', ldap_user.uid], input='y\n')
+        cli, ['members', 'pwreset', uid], input='y\n')
     expected_pat = re.compile((
-        f"^Are you sure you want to reset {ldap_user.uid}'s password\\? \\[y/N\\]: y\n"
+        f"^Are you sure you want to reset {uid}'s password\\? \\[y/N\\]: y\n"
         "New password: \\S+\n$"
     ), re.MULTILINE)
     assert result.exit_code == 0

tests/ceo/cli/test_positions.py

@@ -39,22 +39,29 @@ def test_positions(cli_setup, g_admin_ctx):
     assert result.exit_code == 0
     assert result.output == '''
 The positions will be updated:
-president:      test_0
+president:        test_0
-vice-president: test_1
+vice-president:   test_1
-sysadmin:       test_2
+sysadmin:         test_2
-secretary:      test_3
+secretary:        test_3
-webmaster:      test_4
+webmaster:        test_4
-treasurer:      
+treasurer:
-cro:            
+cro:
-librarian:      
+offsck:
-imapd:          
+ext-affairs-lead:
-offsck:         
+marketing-lead:
+design-lead:
+events-lead:
+reps-lead:
+mods-lead:
+photography-lead:
+codey-bot-lead:
+other:
 Do you want to continue? [y/N]: y
 Update positions in LDAP... Done
 Update executive group in LDAP... Done
 Subscribe to mailing lists... Done
 Transaction successfully completed.
-'''[1:]  # noqa: W291
+'''[1:]
 
     result = runner.invoke(cli, ['positions', 'get'])
     assert result.exit_code == 0
@@ -71,3 +78,78 @@ webmaster:      test_4
         for user in users:
             user.remove_from_ldap()
         group.remove_from_ldap()
+
+
+def test_positions_multiple_users(cli_setup, g_admin_ctx):
+    runner = CliRunner()
+
+    # Setup test data
+    users = []
+    with g_admin_ctx():
+        for i in range(5):
+            user = User(
+                uid=f'test_{i}',
+                cn=f'Test {i}',
+                given_name='Test',
+                sn=str(i),
+                program='Math',
+                terms=['w2023'],
+            )
+            user.add_to_ldap()
+            users.append(user)
+        group = Group(
+            cn='exec',
+            description='Test Group',
+            gid_number=10500,
+        )
+        group.add_to_ldap()
+
+    result = runner.invoke(cli, [
+        'positions', 'set',
+        '--president', 'test_0',
+        '--vice-president', 'test_1,test_2',
+        '--sysadmin', 'test_2',
+        '--secretary', 'test_3, test_4, test_2',
+    ], input='y\n')
+
+    assert result.exit_code == 0
+    assert result.output == '''
+The positions will be updated:
+president:        test_0
+vice-president:   test_1, test_2
+sysadmin:         test_2
+secretary:        test_3, test_4, test_2
+treasurer:
+cro:
+webmaster:
+offsck:
+ext-affairs-lead:
+marketing-lead:
+design-lead:
+events-lead:
+reps-lead:
+mods-lead:
+photography-lead:
+codey-bot-lead:
+other:
+Do you want to continue? [y/N]: y
+Update positions in LDAP... Done
+Update executive group in LDAP... Done
+Subscribe to mailing lists... Done
+Transaction successfully completed.
+'''[1:]
+
+    result = runner.invoke(cli, ['positions', 'get'])
+    assert result.exit_code == 0
+    assert result.output == '''
+president:      test_0
+secretary:      test_2, test_3, test_4
+sysadmin:       test_2
+vice-president: test_1, test_2
+'''[1:]
+
+    # Cleanup test data
+    with g_admin_ctx():
+        for user in users:
+            user.remove_from_ldap()
+        group.remove_from_ldap()

tests/ceo_common/test_utils.py

@@ -0,0 +1,19 @@
+import ceo_common.utils as utils
+
+
+def test_validate_username():
+    assert utils.validate_username('') == utils.UsernameValidationResult(False, 'Username must not be empty')
+    assert utils.validate_username('-failure') == utils.UsernameValidationResult(False, 'Username is invalid')
+    assert utils.validate_username('35 - joe') == utils.UsernameValidationResult(False, 'Username is invalid')
+    assert utils.validate_username('35 -joe') == utils.UsernameValidationResult(False, 'Username is invalid')
+    assert utils.validate_username('35- joe') == utils.UsernameValidationResult(False, 'Username is invalid')
+    assert utils.validate_username('35joe-') == utils.UsernameValidationResult(False, 'Username is invalid')
+    assert utils.validate_username('35$joe') == utils.UsernameValidationResult(False, 'Username is invalid')
+    assert utils.validate_username('35-joe') == utils.UsernameValidationResult(False, 'Username is invalid')
+    assert utils.validate_username(' 35joe') == utils.UsernameValidationResult(False, 'Username is invalid')
+    assert utils.validate_username('35 joe') == utils.UsernameValidationResult(False, 'Username is invalid')
+    assert utils.validate_username('35joe ') == utils.UsernameValidationResult(False, 'Username is invalid')
+    assert utils.validate_username('joe!') == utils.UsernameValidationResult(False, 'Username is invalid')
+    assert utils.validate_username('e45jong') == utils.UsernameValidationResult(True)
+    assert utils.validate_username('joe-35') == utils.UsernameValidationResult(True)
+    assert utils.validate_username('joe35-') == utils.UsernameValidationResult(True)

tests/ceo_dev.ini

@@ -14,8 +14,10 @@ port = 9987
 
 [positions]
 required = president,vice-president,sysadmin
-available = president,vice-president,treasurer,secretary,
+available = president,vice-president,sysadmin,treasurer,
-            sysadmin,cro,librarian,imapd,webmaster,offsck
+            secretary,cro,webmaster,offsck,ext-affairs-lead,
+            marketing-lead,design-lead,events-lead,reps-lead,
+            mods-lead,photography-lead,codey-bot-lead,other
 
 [mysql]
 host = coffee

tests/ceod/api/test_db_mysql.py

@@ -5,8 +5,8 @@ from mysql.connector import connect
 from mysql.connector.errors import ProgrammingError
 
 
-def test_api_create_mysql_db(cfg, client, g_admin_ctx, ldap_user, krb_user):
+def test_api_create_mysql_db(cfg, client, g_admin_ctx, ldap_and_krb_user):
-    uid = ldap_user.uid
+    uid = ldap_and_krb_user.uid
 
     with g_admin_ctx():
         user = User(uid='someone_else', cn='Some Name', given_name='Some',
@@ -72,8 +72,8 @@ def test_api_create_mysql_db(cfg, client, g_admin_ctx, ldap_user, krb_user):
         user.remove_from_ldap()
 
 
-def test_api_passwd_reset_mysql(cfg, client, g_admin_ctx, ldap_user, krb_user):
+def test_api_passwd_reset_mysql(cfg, client, g_admin_ctx, ldap_and_krb_user):
-    uid = ldap_user.uid
+    uid = ldap_and_krb_user.uid
 
     with g_admin_ctx():
         user = User(uid='someone_else', cn='Some Name', given_name='Some',

tests/ceod/api/test_db_psql.py

@@ -4,8 +4,8 @@ from ceod.model import User
 from psycopg2 import connect, OperationalError, ProgrammingError
 
 
-def test_api_create_psql_db(cfg, client, g_admin_ctx, ldap_user, krb_user):
+def test_api_create_psql_db(cfg, client, g_admin_ctx, ldap_and_krb_user):
-    uid = ldap_user.uid
+    uid = ldap_and_krb_user.uid
 
     with g_admin_ctx():
         user = User(uid='someone_else', cn='Some Name', given_name='Some',
@@ -74,8 +74,8 @@ def test_api_create_psql_db(cfg, client, g_admin_ctx, ldap_user, krb_user):
         user.remove_from_ldap()
 
 
-def test_api_passwd_reset_psql(cfg, client, g_admin_ctx, ldap_user, krb_user):
+def test_api_passwd_reset_psql(cfg, client, g_admin_ctx, ldap_and_krb_user):
-    uid = ldap_user.uid
+    uid = ldap_and_krb_user.uid
 
     with g_admin_ctx():
         user = User(uid='someone_else', cn='Some Name', given_name='Some',

tests/ceod/api/test_groups.py

@@ -2,6 +2,7 @@ import ldap3
 import pytest
 
 from ceod.model import Group
+from tests.conftest_ceod_api import expect_successful_txn
 
 
 def test_api_group_not_found(client):
@@ -9,20 +10,28 @@ def test_api_group_not_found(client):
     assert status == 404
 
 
-@pytest.fixture(scope='module')
+def create_group(client, cn: str, description: str):
-def create_group_resp(client):
     status, data = client.post('/api/groups', json={
-        'cn': 'test_group1',
+        'cn': cn,
-        'description': 'Test Group One',
+        'description': description,
     })
     assert status == 200
     assert data[-1]['status'] == 'completed'
-    yield status, data
+    return status, data
-    status, data = client.delete('/api/groups/test_group1')
+
+
+def delete_group(client, cn: str):
+    status, data = client.delete(f'/api/groups/{cn}')
     assert status == 200
     assert data[-1]['status'] == 'completed'
 
 
+@pytest.fixture(scope='module')
+def create_group_resp(client):
+    yield create_group(client, 'test_group1', 'Test Group One')
+    delete_group(client, 'test_group1')
+
+
 @pytest.fixture(scope='module')
 def create_group_result(create_group_resp):
     # convenience method
@@ -111,6 +120,47 @@ def test_api_add_member_to_group(client, create_group_result, ldap_user):
     assert data['members'] == []
 
 
+@pytest.fixture(scope='module')
+def create_random_names():
+    # generated with https://www.randomlists.com/random-words
+    random_names = [
+        "intelligent",
+        "skin",
+        "shivering",
+    ]
+    yield random_names
+
+
+@pytest.fixture(scope='module')
+def create_searchable_groups(client, create_random_names):
+    random_names = create_random_names
+    for name in random_names:
+        create_group(client, name, 'Groups with distinct names for testing searching')
+    yield random_names
+
+
+def test_api_group_search(client, create_searchable_groups):
+    cns = create_searchable_groups
+    # pairs of cn indices as well as amount of results that should be returned
+    random_numbers = [
+        (0, 68),
+        (1, 54),
+        (2, 97),
+    ]
+
+    for tup in random_numbers:
+        cn = cns[tup[0]]
+        status, data = client.get(f'/api/groups/search/{cn}/{tup[1]}')
+        assert status == 200
+        assert len(data) == tup[1]
+        assert data[0] == cn
+
+    for cn in cns:
+        status, data = client.delete(f'/api/groups/{cn}')
+        assert status == 200
+        assert data[-1]['status'] == 'completed'
+
+
 def test_api_group_auxiliary(cfg, client, ldap_user, g_admin_ctx):
     # Make sure that syscom has auxiliary mailing lists and groups
     # defined in ceod_test_local.ini.
@@ -164,3 +214,27 @@ def test_api_group_auxiliary(cfg, client, ldap_user, g_admin_ctx):
         for group in groups:
             if group.cn != 'syscom':
                 group.remove_from_ldap()
+
+
+@pytest.mark.parametrize('desc,expect_success', [
+    ('in_syscom', True),
+    ('is_offsck', True),
+    (None, False),
+])
+def test_api_add_or_remove_member_from_office(desc, expect_success, cfg, client, ldap_and_krb_user, new_user, g_admin_ctx):
+    uid = ldap_and_krb_user.uid
+    if desc == 'in_syscom':
+        uid = 'ctdalek'
+    elif desc == 'is_offsck':
+        with g_admin_ctx():
+            ldap_and_krb_user.set_positions(['offsck'])
+
+    def handle_resp(status_and_data):
+        if expect_success:
+            expect_successful_txn(status_and_data)
+        else:
+            status = status_and_data[0]
+            assert status == 403
+
+    handle_resp(client.post(f'/api/groups/office/members/{new_user.uid}', principal=uid))
+    handle_resp(client.delete(f'/api/groups/office/members/{new_user.uid}', principal=uid))

tests/ceod/api/test_members.py

@@ -111,6 +111,7 @@ def test_api_next_uid(cfg, client, create_user_result):
         'sn': 'Two',
         'program': 'Math',
         'terms': ['s2021'],
+        'forwarding_addresses': ['test2@uwaterloo.internal']
     })
     assert data[-1]['status'] == 'completed'
     result = data[-1]['result']
@@ -121,12 +122,60 @@ def test_api_next_uid(cfg, client, create_user_result):
         client.delete('/api/members/test2')
 
 
+def test_api_create_user_without_forwarding_addresses(cfg, client):
+    status, data = client.post('/api/members', json={
+        'uid': 'test3',
+        'cn': 'Test Three',
+        'given_name': 'Test',
+        'sn': 'Three',
+        'program': 'Math',
+        'terms': ['s2023'],
+    })
+    assert status == 400
+    assert data['error'] == "BadRequest: Attribute 'forwarding_addresses' is missing or empty"
+
+
+def test_api_create_user_without_valid_username(cfg, client):
+    status, data = client.post('/api/members', json={
+        'uid': '4_test',
+        'cn': 'Test Four',
+        'given_name': 'Test',
+        'sn': 'Four',
+        'program': 'Math',
+        'terms': ['w2024'],
+        'forwarding_addresses': ['test4@uwaterloo.internal'],
+    })
+    try:
+        assert status == 400
+        assert data['error'] == "BadRequest: Attribute 'uid' is missing or invalid"
+    finally:
+        client.delete('/api/members/4_test')
+
+
+def test_api_create_user_with_valid_username(cfg, client):
+    status, data = client.post('/api/members', json={
+        'uid': 'test-4',
+        'cn': 'Test Four',
+        'given_name': 'Test',
+        'sn': 'Four',
+        'program': 'Math',
+        'terms': ['w2024'],
+        'forwarding_addresses': ['test4@uwaterloo.internal'],
+    })
+    try:
+        assert status == 200
+        assert data[-1]['status'] == 'completed'
+    finally:
+        client.delete('/api/members/test-4')
+
+
 def test_api_get_user(cfg, client, create_user_result):
     old_data = create_user_result.copy()
     uid = old_data['uid']
     del old_data['password']
 
     status, data = client.get(f'/api/members/{uid}')
+    del data['groups']
     assert status == 200
     assert data == old_data
 
@@ -251,8 +300,8 @@ def test_api_reset_password(client, create_user_result):
 def test_authz_check(client, create_user_result):
     # non-staff members may not create users
     status, data = client.post('/api/members', json={
-        'uid': 'test1', 'cn': 'Test One', 'given_name': 'Test',
+        'uid': 'test1', 'cn': 'Test One', 'given_name': 'Test', 'sn': 'One',
-        'sn': 'One', 'terms': ['s2021'],
+        'terms': ['s2021'], 'forwarding_addresses': ['test1@uwaterloo.internal']
     }, principal='regular1')
     assert status == 403
 
@@ -262,12 +311,13 @@ def test_authz_check(client, create_user_result):
     del old_data['password']
     del old_data['forwarding_addresses']
     _, data = client.get(f'/api/members/{uid}', principal='regular1')
+    del data['groups']
     assert data == old_data
 
     # If we're syscom but we don't pass credentials, the request should fail
     _, data = client.post('/api/members', json={
-        'uid': 'test1', 'cn': 'Test One', 'given_name': 'Test',
+        'uid': 'test1', 'cn': 'Test One', 'given_name': 'Test', 'sn': 'One',
-        'sn': 'One', 'terms': ['s2021'],
+        'terms': ['s2021'], 'forwarding_addresses': ['test1@uwaterloo.internal']
     }, principal='ctdalek', delegate=False)
     assert data[-1]['status'] == 'aborted'
 

tests/ceod/api/test_positions.py

@@ -7,8 +7,8 @@ def test_get_positions(client, ldap_user, g_admin_ctx):
     status, data = client.get('/api/positions')
     assert status == 200
     expected = {
-        'president': ldap_user.uid,
+        'president': [ldap_user.uid],
-        'treasurer': ldap_user.uid,
+        'treasurer': [ldap_user.uid],
     }
     assert data == expected
 
@@ -20,7 +20,7 @@ def test_set_positions(cfg, client, g_admin_ctx, mock_mailman_server):
 
     users = []
     with g_admin_ctx():
-        for uid in ['test_1', 'test_2', 'test_3', 'test_4']:
+        for uid in ['test1', 'test2', 'test3', 'test4']:
             user = User(uid=uid, cn='Some Name', given_name='Some', sn='Name',
                         terms=['s2021'])
             user.add_to_ldap()
@@ -31,23 +31,23 @@ def test_set_positions(cfg, client, g_admin_ctx, mock_mailman_server):
     try:
         # missing required position
         status, _ = client.post('/api/positions', json={
-            'vice-president': 'test_1',
+            'vice-president': 'test1',
         })
         assert status == 400
 
         # non-existent position
         status, _ = client.post('/api/positions', json={
-            'president': 'test_1',
+            'president': 'test1',
-            'vice-president': 'test_2',
+            'vice-president': 'test2',
-            'sysadmin': 'test_3',
+            'sysadmin': 'test3',
-            'no-such-position': 'test_3',
+            'no-such-position': 'test3',
         })
         assert status == 400
 
         status, data = client.post('/api/positions', json={
-            'president': 'test_1',
+            'president': 'test1',
-            'vice-president': 'test_2',
+            'vice-president': 'test2',
-            'sysadmin': 'test_3',
+            'sysadmin': 'test3',
         })
         assert status == 200
         expected = [
@@ -55,16 +55,16 @@ def test_set_positions(cfg, client, g_admin_ctx, mock_mailman_server):
             {"status": "in progress", "operation": "update_exec_group_ldap"},
             {"status": "in progress", "operation": "subscribe_to_mailing_lists"},
             {"status": "completed", "result": {
-                "president": "test_1",
+                "president": ["test1"],
-                "vice-president": "test_2",
+                "vice-president": ["test2"],
-                "sysadmin": "test_3",
+                "sysadmin": ["test3"],
             }},
         ]
         assert data == expected
         # make sure execs were added to exec group
         status, data = client.get('/api/groups/exec')
         assert status == 200
-        expected = ['test_1', 'test_2', 'test_3']
+        expected = ['test1', 'test2', 'test3']
         assert sorted([item['uid'] for item in data['members']]) == expected
         # make sure execs were subscribed to mailing lists
         addresses = [f'{uid}@{base_domain}' for uid in expected]
@@ -72,20 +72,55 @@ def test_set_positions(cfg, client, g_admin_ctx, mock_mailman_server):
             assert sorted(mock_mailman_server.subscriptions[mailing_list]) == addresses
 
         _, data = client.post('/api/positions', json={
-            'president': 'test_1',
+            'president': 'test1',
-            'vice-president': 'test_2',
+            'vice-president': 'test2',
-            'sysadmin': 'test_2',
+            'sysadmin': 'test2',
-            'treasurer': 'test_4',
+            'treasurer': 'test4',
         })
         assert data[-1]['status'] == 'completed'
         # make sure old exec was removed from group
-        expected = ['test_1', 'test_2', 'test_4']
+        expected = ['test1', 'test2', 'test4']
         _, data = client.get('/api/groups/exec')
         assert sorted([item['uid'] for item in data['members']]) == expected
         # make sure old exec was removed from mailing lists
         addresses = [f'{uid}@{base_domain}' for uid in expected]
         for mailing_list in mailing_lists:
             assert sorted(mock_mailman_server.subscriptions[mailing_list]) == addresses
+
+        # multiple users per position
+        status, data = client.post('/api/positions', json={
+            'president': 'test1',
+            'vice-president': ['test2', 'test3'],
+            'sysadmin': 'test2, test3,test4',
+        })
+        assert status == 200
+        assert data[-1] == {'status': 'completed', 'result': {
+            'president': ['test1'],
+            'vice-president': ['test2', 'test3'],
+            'sysadmin': ['test2', 'test3', 'test4'],
+        }}
+        # check every single psition
+        status, data = client.post('/api/positions', json={
+            'president': 'test1',
+            'vice-president': 'test1',
+            'treasurer': 'test1',
+            'secretary': 'test1',
+            'sysadmin': 'test1',
+            'cro': 'test1',
+            'webmaster': 'test1',
+            'offsck': 'test1',
+            'ext-affairs-lead': 'test1',
+            'marketing-lead': 'test1',
+            'design-lead': 'test1',
+            'events-lead': 'test1',
+            'reps-lead': 'test1',
+            'mods-lead': 'test1',
+            'photography-lead': 'test1',
+            'codey-bot-lead': 'test1',
+            'other': 'test1',
+        })
+        assert status == 200
+        assert data[-1]['status'] == 'completed'
     finally:
         with g_admin_ctx():
             for user in users:

tests/ceod/api/test_uwldap.py

@@ -50,3 +50,19 @@ def test_updateprograms(
     # make sure that the user was changed
     status, data = client.get(f'/api/members/{uwldap_user.uid}')
     assert data['program'] == 'New Program'
+
+
+def test_get_alumni(client):
+    uid = 'alumni1'
+    status, data = client.get(f'/api/uwldap/{uid}')
+    assert status == 200
+    expected = {
+        "cn": 'Alumni One',
+        "given_name": 'Alumni',
+        "sn": 'One',
+        # This should be the email address from AD LDAP
+        "mail_local_addresses": [f'{uid}@alumni.uwaterloo.internal'],
+        # Program attribute should be missing
+        "uid": uid,
+    }
+    assert data == expected

tests/ceod_dev.ini

@@ -27,6 +27,10 @@ sudo_base = ou=SUDOers,dc=csclub,dc=internal
 server_url = ldap://auth1.csclub.internal
 base = ou=UWLDAP,dc=csclub,dc=internal
 
+[adldap]
+server_url = ldap://auth1.csclub.internal
+base = ou=ADLDAP,dc=csclub,dc=internal
+
 [members]
 min_id = 20001
 max_id = 29999
@@ -59,8 +63,10 @@ exec = exec
 
 [positions]
 required = president,vice-president,sysadmin
-available = president,vice-president,treasurer,secretary,
+available = president,vice-president,sysadmin,treasurer,
-            sysadmin,cro,librarian,imapd,webmaster,offsck
+            secretary,cro,webmaster,offsck,ext-affairs-lead,
+            marketing-lead,design-lead,events-lead,reps-lead,
+            mods-lead,photography-lead,codey-bot-lead,other
 
 [mysql]
 username = mysql
@@ -91,6 +97,7 @@ members_domain = csclub.cloud
 k8s_members_domain = k8s.csclub.cloud
 ip_range_min = 172.19.134.10
 ip_range_max = 172.19.134.160
+reload_web_server_cmd = systemctl reload nginx
 
 [k8s]
 members_clusterrole = csc-members-default

tests/ceod_test_local.ini

@@ -26,6 +26,10 @@ sudo_base = ou=TestSUDOers,dc=csclub,dc=internal
 server_url = ldap://auth1.csclub.internal
 base = ou=TestUWLDAP,dc=csclub,dc=internal
 
+[adldap]
+server_url = ldap://auth1.csclub.internal
+base = ou=TestADLDAP,dc=csclub,dc=internal
+
 [members]
 # 20000 is ctdalek, 20001 is office1
 min_id = 20002
@@ -58,8 +62,10 @@ exec = exec
 
 [positions]
 required = president,vice-president,sysadmin
-available = president,vice-president,treasurer,secretary,
+available = president,vice-president,sysadmin,treasurer,
-            sysadmin,cro,librarian,imapd,webmaster,offsck
+            secretary,cro,webmaster,offsck,ext-affairs-lead,
+            marketing-lead,design-lead,events-lead,reps-lead,
+            mods-lead,photography-lead,codey-bot-lead,other
 
 [mysql]
 username = mysql
@@ -90,6 +96,7 @@ members_domain = csclub.cloud
 k8s_members_domain = k8s.csclub.cloud
 ip_range_min = 172.19.134.10
 ip_range_max = 172.19.134.160
+reload_web_server_cmd = systemctl reload nginx
 
 [k8s]
 members_clusterrole = csc-members-default

tests/conftest.py

@@ -30,15 +30,16 @@ from .utils import (  # noqa: F401
 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
     IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
     IDatabaseService, ICloudStackService, IKubernetesService, IVHostManager, \
-    ICloudResourceManager, IContainerRegistryService, IClubWebHostingService
+    ICloudResourceManager, IContainerRegistryService, IClubWebHostingService, \
-from ceo_common.model import Config, HTTPClient, Term
+    IADLDAPService
+from ceo_common.model import Config, HTTPClient, Term, UWLDAPRecord
 import ceo_common.utils
 from ceod.api import create_app
 from ceod.db import MySQLService, PostgreSQLService
 from ceod.model import KerberosService, LDAPService, FileService, User, \
-    MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \
+    MailmanService, Group, UWLDAPService, MailService, \
     CloudStackService, KubernetesService, VHostManager, CloudResourceManager, \
-    ContainerRegistryService, ClubWebHostingService
+    ContainerRegistryService, ClubWebHostingService, ADLDAPService
 from .MockSMTPServer import MockSMTPServer
 from .MockMailmanServer import MockMailmanServer
 from .MockCloudStackServer import MockCloudStackServer
@@ -298,6 +299,28 @@ def uwldap_srv(cfg, ldap_conn):
     delete_subtree(conn, base_dn)
 
     conn.add(base_dn, 'organizationalUnit')
+    conn.add(
+        f'uid=ctdalek,{base_dn}',
+        ['inetLocalMailRecipient', 'inetOrgPerson', 'organizationalPerson', 'person'],
+        {
+            'mailLocalAddress': 'ctdalek@uwaterloo.internal',
+            'ou': 'Math',
+            'cn': 'Calum T. Dalek',
+            'sn': 'Dalek',
+            'givenName': 'Calum',
+        },
+    )
+    conn.add(
+        f'uid=alumni1,{base_dn}',
+        ['inetLocalMailRecipient', 'inetOrgPerson', 'organizationalPerson', 'person'],
+        {
+            'mailLocalAddress': 'alumni1@uwaterloo.internal',
+            'ou': 'Alumni',
+            'cn': 'Alumni',
+            'sn': 'One',
+            'givenName': 'Alumni',
+        },
+    )
     _uwldap_srv = UWLDAPService()
     component.getGlobalSiteManager().registerUtility(_uwldap_srv, IUWLDAPService)
     yield _uwldap_srv
@@ -305,6 +328,31 @@ def uwldap_srv(cfg, ldap_conn):
     delete_subtree(conn, base_dn)
 
 
+@pytest.fixture(scope='session')
+def adldap_srv(cfg, ldap_conn):
+    conn = ldap_conn
+    base_dn = cfg.get('adldap_base')
+    delete_subtree(conn, base_dn)
+    conn.add(base_dn, 'organizationalUnit')
+    conn.add(
+        f'cn=alumni1,{base_dn}',
+        ['mockADUser', 'inetOrgPerson', 'organizationalPerson', 'person'],
+        {
+            'description': 'One, Alumni',
+            'givenName': 'Alumni',
+            'sn': '*One',
+            'cn': 'alumni1',
+            'sAMAccountName': 'alumni1',
+            'displayName': 'alumni1',
+            'mail': 'alumni1@alumni.uwaterloo.internal',
+        },
+    )
+    _adldap_srv = ADLDAPService()
+    component.getGlobalSiteManager().registerUtility(_adldap_srv, IADLDAPService)
+    yield _adldap_srv
+    delete_subtree(conn, base_dn)
+
+
 @pytest.fixture(scope='session')
 def webhosting_srv():
     with tempfile.TemporaryDirectory() as tmpdir:
@@ -365,14 +413,25 @@ def postgresql_srv(cfg):
     return psql_srv
 
 
+def delete_dir_contents(dir: str):
+    if os.path.isdir(dir):
+        for item in os.listdir(dir):
+            path = os.path.join(dir, item)
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+            else:
+                os.unlink(path)
+
+
 @pytest.fixture(scope='session')
 def vhost_dir_setup(cfg):
     state_dir = '/run/ceod'
-    if os.path.isdir(state_dir):
+    # Don't delete the directory itself because the non-test ceod process
-        shutil.rmtree(state_dir)
+    # is using it
-    os.makedirs(state_dir)
+    delete_dir_contents(state_dir)
+    os.makedirs(state_dir, exist_ok=True)
     yield
-    shutil.rmtree(state_dir)
+    delete_dir_contents(state_dir)
 
 
 @pytest.fixture(scope='session')
@@ -411,6 +470,7 @@ def app(
     file_srv,
     mailman_srv,
     uwldap_srv,
+    adldap_srv,
     mail_srv,
     mysql_srv,
     postgresql_srv,
@@ -478,6 +538,12 @@ def krb_user(simple_user):
     simple_user.remove_from_kerberos()
 
 
+@pytest.fixture
+def ldap_and_krb_user(ldap_user, krb_user):
+    # Note that ldap_user and krb_user are both the same person (simple_user)
+    yield ldap_user
+
+
 @pytest.fixture
 def new_user_gen(
     client, g_admin_ctx, ldap_srv_session, mocks_for_create_user,  # noqa: F811
@@ -496,6 +562,7 @@ def new_user_gen(
             'sn': 'Doe',
             'program': 'Math',
             'terms': [str(Term.current())],
+            'forwarding_addresses': [f'{uid}@uwaterloo.internal']
         })
         assert status == 200
         assert data[-1]['status'] == 'completed'

tests/conftest_ceod_api.py

@@ -1,5 +1,6 @@
 import json
 import socket
+from typing import Any, Dict, List, Tuple
 
 import flask
 from flask.testing import FlaskClient
@@ -76,3 +77,9 @@ class CeodTestClient:
 
     def put(self, path, principal=None, need_auth=True, delegate=True, **kwargs):
         return self.request('PUT', path, principal, need_auth, delegate, **kwargs)
+
+
+def expect_successful_txn(status_and_data: Tuple[int, List[Dict[str, Any]]]) -> None:
+    status, data = status_and_data
+    assert status == 200
+    assert data[-1]['status'] == 'completed'

web/.gitignore

@@ -0,0 +1,3 @@
+/ceod-web
+/app.sock
+/test

web/README.md

@@ -0,0 +1,80 @@
+# ceod-web
+This directory contains an experimental self-service web portal for CSC
+members to use. It is an alternative to the ceo TUI. Currently it is only
+meant to be used by general members, but in the future it may be extended
+to be used by syscom/office members as well.
+
+Implemented APIs:
+- [x] Password reset
+- [ ] Change login shell
+- [ ] Change forwarding addresses
+- [ ] Show membership terms
+
+## Development
+Make sure the Docker containers for ceo are running. Build the "web"
+executable on the host, then run it in the phosphoric-acid container:
+```sh
+# Don't use cgo because the glibc version in the container will likely be
+# older than the one on the host
+CGO_ENABLED=0 go build -o ceod-web
+docker-compose exec phosphoric-acid bash
+cd web
+./ceod-web -c dev.json
+```
+
+The application will listen on a Unix socket. In production, it expects to
+receive ADFS information from Apache, which is acting as a reverse proxy.
+In development, we will use our own proxy instead:
+```sh
+# On the host
+go run scripts/proxy.go -s app.sock -u ctdalek -f Calum
+```
+Now you should be able to visit http://localhost:9988 in your browser.
+
+NOTE: If you are not accessing the website via "localhost" (e.g. you are using
+some custom proxy setup), then you need to modify the value of "app_url" in
+dev.json. The app_url value must be equal to the base URL which you enter in
+your browser's address bar, otherwise the cookie domain will not match.
+
+You can change the `-u` (username) and `-f` (first name) arguments to the proxy.go
+program to simulate a different user. See the .drone/data.ldif file in the parent
+directory to see all mock users.
+
+### Templates and static assets
+In development, the templated views and static assets will be loaded from the
+internal/views and internal/static folders, respectively. Unfortunately I think
+that the Echo framework caches those files internally so you will need to
+restart the app process if you modify them.
+
+### Emails
+By default, no emails will be sent; they will only be printed to stdout. To
+send real emails, add these fields to the dev.json (replace `your_username`):
+```json
+{
+  ...
+  "mta": "mail.csclub.uwaterloo.ca:25",
+  "forced_email_recipient": "your_username@csclub.uwaterloo.ca"
+}
+```
+This will send all of the emails to your email address (the `To` header will
+still be preserved, however).
+
+## Tests
+```sh
+# On the host
+CGO_ENABLED=0 go test -c -o test ./tests
+# In the container
+./test -test.v
+```
+
+## Deployment
+Apache configuration on caffeine:
+```
+Redirect permanent /ceo /ceo/
+<Location /ceo/ >
+  Include snippets/adfs-require-auth.conf
+  Include snippets/adfs-set-headers.conf
+  ProxyPass        "unix:/run/ceod-web/app.sock|http://ceod-web/"
+  ProxyPassReverse "unix:/run/ceod-web/app.sock|http://ceod-web/"
+</Location>
+```

web/dev.json

@@ -0,0 +1,7 @@
+{
+  "app_url": "http://localhost:9988",
+  "socket_path": "app.sock",
+  "csc_domain": "csclub.internal",
+  "uw_domain": "uwaterloo.internal",
+  "dev": true
+}

web/go.mod

@@ -0,0 +1,34 @@
+module git.csclub.uwaterloo.ca/public/pyceo/web
+
+go 1.21.4
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.1
+	github.com/jcmturner/gokrb5/v8 v8.4.4
+	github.com/labstack/echo/v4 v4.11.4
+	github.com/labstack/gommon v0.4.2
+	github.com/stretchr/testify v1.8.4
+)
+
+require (
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/hashicorp/go-uuid v1.0.3 // indirect
+	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
+	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
+	github.com/jcmturner/gofork v1.7.6 // indirect
+	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
+	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasttemplate v1.2.2 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

web/go.sum

@@ -0,0 +1,103 @@
+github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
+github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
+github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=
+github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
+github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
+github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
+github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
+github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
+github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
+github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=
+github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
+github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
+github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
+github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=
+github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
+github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
+github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
+github.com/labstack/echo/v4 v4.11.4 h1:vDZmA+qNeh1pd/cCkEicDMrjtrnMGQ1QFI9gWN1zGq8=
+github.com/labstack/echo/v4 v4.11.4/go.mod h1:noh7EvLwqDsmh/X/HWKPUl1AjzJrhyptRyEbQJfxen8=
+github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
+github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
+github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

web/internal/api/api.go

@@ -0,0 +1,180 @@
+package api
+
+import (
+	"fmt"
+	"html/template"
+	"io"
+	"io/fs"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+
+	"github.com/labstack/echo/v4"
+	"github.com/labstack/echo/v4/middleware"
+	"github.com/labstack/gommon/log"
+
+	"git.csclub.uwaterloo.ca/public/pyceo/web/internal"
+	"git.csclub.uwaterloo.ca/public/pyceo/web/internal/app"
+	"git.csclub.uwaterloo.ca/public/pyceo/web/internal/config"
+	"git.csclub.uwaterloo.ca/public/pyceo/web/internal/service"
+	"git.csclub.uwaterloo.ca/public/pyceo/web/pkg/model"
+)
+
+type TemplateManager struct {
+	templates map[string]*template.Template
+}
+
+func newTemplateManager(isDev bool) *TemplateManager {
+	var viewsFS fs.FS
+	if isDev {
+		viewsFS = os.DirFS("internal/views")
+	} else {
+		var err error
+		viewsFS, err = fs.Sub(internal.EmbeddedViews, "views")
+		if err != nil {
+			panic(err)
+		}
+	}
+	base := template.Must(template.ParseFS(viewsFS, "base.html"))
+	// Adapted from https://stackoverflow.com/a/24120195
+	parse := func(filenames ...string) *template.Template {
+		clone := template.Must(base.Clone())
+		return template.Must(clone.ParseFS(viewsFS, filenames...))
+	}
+	return &TemplateManager{templates: map[string]*template.Template{
+		"root":                 parse("root.html"),
+		"pwreset":              parse("pwreset.html"),
+		"pwreset_confirmation": parse("pwreset_confirmation.html"),
+		"app_error":            parse("app_error.html"),
+		"4xx":                  parse("4xx.html"),
+		"500":                  parse("500.html"),
+	}}
+}
+
+func (t *TemplateManager) Render(w io.Writer, name string, data any, c echo.Context) error {
+	tmpl, ok := t.templates[name]
+	if !ok {
+		c.Logger().Errorf("No such template '%s'", name)
+		return echo.NewHTTPError(http.StatusInternalServerError, "Internal server error")
+	}
+	return tmpl.Execute(w, data)
+}
+
+func addStaticAssets(isDev bool, e *echo.Echo) {
+	var staticFS fs.FS
+	if isDev {
+		staticFS = os.DirFS("internal")
+	} else {
+		staticFS = internal.EmbeddedAssets
+	}
+	e.Group("static").Use(middleware.StaticWithConfig(middleware.StaticConfig{
+		Root:       "static",
+		Filesystem: http.FS(staticFS),
+	}))
+}
+
+func newListener(cfg *config.Config) net.Listener {
+	sockPath := cfg.SocketPath
+	if _, err := os.Stat(sockPath); err == nil {
+		err = os.Remove(sockPath)
+		if err != nil {
+			panic(fmt.Errorf("Could not remove %s: %w", sockPath, err))
+		}
+	}
+	l, err := net.Listen("unix", sockPath)
+	if err != nil {
+		panic(err)
+	}
+	if !cfg.IsDev && !cfg.NoSocketAuth {
+		// Only www-data should be allowed to write to the socket, otherwise
+		// users could forge the X-CSC-ADFS-* headers and reset other people's
+		// passwords
+		err = os.Chmod(sockPath, 0)
+		if err != nil {
+			panic(err)
+		}
+		cmd := exec.Command("/bin/setfacl", "-m", "u:www-data:rw", sockPath)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			panic(err)
+		}
+	}
+	return l
+}
+
+func NewAPI(cfg *config.Config, ceodSrv model.CeodService, mailSrv service.MailService) *echo.Echo {
+	e := echo.New()
+	app := app.NewApp(cfg, mailSrv)
+	e.Logger.SetLevel(log.DEBUG)
+	e.Use(middleware.Logger())
+	e.Use(middleware.Recover())
+	e.Use(middleware.RequestID())
+	e.Use(helmet(cfg))
+	addStaticAssets(cfg.IsDev, e)
+	e.Use(appContextMiddleware(app))
+	e.Use(middleware.CSRFWithConfig(middleware.CSRFConfig{
+		TokenLookup:    "cookie:" + cfg.CookieName,
+		CookieName:     "ceod-web-csrf",
+		CookiePath:     cfg.GetAppPath(),
+		CookieDomain:   cfg.GetAppHostname(),
+		CookieSecure:   !cfg.IsDev,
+		CookieHTTPOnly: true,
+		CookieSameSite: http.SameSiteStrictMode,
+	}))
+	e.HTTPErrorHandler = httpErrorHandler
+	e.Renderer = newTemplateManager(cfg.IsDev)
+	addRoutes(e)
+	return e
+}
+
+func Start(cfg *config.Config) {
+	e := NewAPI(cfg, service.NewCeodService(cfg), service.NewMailService(cfg))
+	listener := newListener(cfg)
+	e.Logger.Info("Listening on " + cfg.SocketPath)
+	e.Logger.Fatal(http.Serve(listener, e))
+}
+
+func addRoutes(e *echo.Echo) {
+	e.GET("/", getRoot)
+	e.GET("/pwreset", getPwreset)
+	e.POST("/pwreset", postPwreset)
+}
+
+func getRoot(c echo.Context) error {
+	ac := c.(*appContext)
+	_, err := ac.app.GetReqUser(ac)
+	if err != nil {
+		return err
+	}
+	return c.Render(http.StatusOK, "root", map[string]any{"firstName": ac.req.GivenName})
+}
+
+func getPwreset(c echo.Context) error {
+	ac := c.(*appContext)
+	emailAddress, err := ac.app.PwresetCheck(ac)
+	if err != nil {
+		return err
+	}
+	// Double-submit cookie
+	renderData := map[string]any{
+		"emailAddress": emailAddress,
+		"csrf":         c.Get("csrf"),
+	}
+	return c.Render(http.StatusOK, "pwreset", renderData)
+}
+
+func postPwreset(c echo.Context) error {
+	if c.Get("csrf") != c.FormValue("_csrf") {
+		return newApiError(ERROR_INVALID_CSRF_TOKEN)
+	}
+	ac := c.(*appContext)
+	emailAddress, err := ac.app.Pwreset(ac)
+	if err != nil {
+		return err
+	}
+	renderData := map[string]any{"emailAddress": emailAddress}
+	return c.Render(http.StatusOK, "pwreset_confirmation", renderData)
+}

web/internal/api/errors.go

@@ -0,0 +1,99 @@
+package api
+
+import (
+	"html/template"
+	"net/http"
+
+	"github.com/labstack/echo/v4"
+
+	"git.csclub.uwaterloo.ca/public/pyceo/web/internal/app"
+)
+
+const (
+	ERROR_INVALID_CSRF_TOKEN = iota + 1
+)
+const (
+	membershipURL = "https://csclub.uwaterloo.ca/get-involved/"
+	syscomURL     = "mailto:syscom@csclub.uwaterloo.ca"
+)
+
+type ApiError struct {
+	Code int
+}
+
+func newApiError(code int) ApiError {
+	return ApiError{Code: code}
+}
+func (a ApiError) Error() string {
+	return "API error"
+}
+
+func renderHTTPErrorPage(c echo.Context, code int, name string) {
+	data := map[string]any{"statusText": http.StatusText(code)}
+	if err := c.Render(code, name, data); err != nil {
+		c.Logger().Error(err)
+		// Fall back to plain text response
+		_ = c.String(http.StatusInternalServerError, "Internal server error")
+	}
+}
+
+func renderAppErrorPage(c echo.Context, renderInfo *errorRenderInfo) {
+	if err := c.Render(http.StatusOK, "app_error", renderInfo); err != nil {
+		c.Logger().Error(err)
+		// Fall back to plain text response
+		_ = c.String(http.StatusInternalServerError, "Internal server error")
+	}
+}
+
+type errorRenderInfo struct {
+	Title        string
+	HtmlFragment template.HTML
+}
+
+func newErrorRenderInfo(title string, htmlFragment string) *errorRenderInfo {
+	return &errorRenderInfo{Title: title, HtmlFragment: template.HTML(htmlFragment)}
+}
+
+var appErrorInfos = map[int]*errorRenderInfo{
+	app.ERROR_NO_SUCH_USER: newErrorRenderInfo(
+		"You are not a CSC member :(",
+		"It seems like you're not a CSC member yet. "+
+			`Maybe you'd like to <a href="`+membershipURL+`">become one instead</a>?`),
+	app.ERROR_MEMBERSHIP_EXPIRED: newErrorRenderInfo(
+		"Your membership has expired",
+		`Please visit <a href="`+membershipURL+`">this page</a> for instructions `+
+			"on how to renew your membership."),
+	app.ERROR_OTHER: newErrorRenderInfo(
+		"Something went wrong",
+		"We seem to be having issues on our end. Please contact the "+
+			`<a href="`+syscomURL+`">Systems Committee</a> for assistance.`),
+	app.ERROR_FAILED_TO_SEND_PWRESET_EMAIL: newErrorRenderInfo(
+		"Something went wrong",
+		"We weren't able to send the new password to your email address. Please "+
+			`contact the <a href="`+syscomURL+`">Systems Committee</a> for assistance.`),
+}
+
+var apiErrorInfos = map[int]*errorRenderInfo{
+	ERROR_INVALID_CSRF_TOKEN: newErrorRenderInfo(
+		"Invalid CSRF token",
+		"Please go back and refresh the page to get a new token. If this "+
+			`error persists, please contact the <a href="`+syscomURL+`">`+
+			"Systems Committee</a>."),
+}
+
+func httpErrorHandler(err error, c echo.Context) {
+	if httpErr, ok := err.(*echo.HTTPError); ok {
+		if httpErr.Code/100 == 4 {
+			renderHTTPErrorPage(c, httpErr.Code, "4xx")
+			return
+		}
+	} else if appErr, ok := err.(app.AppError); ok {
+		renderAppErrorPage(c, appErrorInfos[appErr.Code])
+		return
+	} else if apiErr, ok := err.(ApiError); ok {
+		renderAppErrorPage(c, apiErrorInfos[apiErr.Code])
+		return
+	}
+	c.Logger().Error(err)
+	renderHTTPErrorPage(c, http.StatusInternalServerError, "500")
+}

web/internal/api/middleware.go

@@ -0,0 +1,93 @@
+package api
+
+import (
+	"errors"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/labstack/echo/v4"
+
+	"git.csclub.uwaterloo.ca/public/pyceo/web/internal/app"
+	"git.csclub.uwaterloo.ca/public/pyceo/web/internal/config"
+	"git.csclub.uwaterloo.ca/public/pyceo/web/pkg/logging"
+)
+
+func helmet(cfg *config.Config) echo.MiddlewareFunc {
+	cspSchemes := "https:"
+	if cfg.IsDev {
+		cspSchemes = "http: https:"
+	}
+	cspDirectives := []string{
+		"default-src 'self'",
+		"base-uri 'self'",
+		"font-src 'self' " + cspSchemes + " data:",
+		"form-action 'self'",
+		"frame-ancestors 'self'",
+		"img-src 'self' data:",
+		"object-src 'none'",
+		"script-src 'self'",
+		"script-src-attr 'none'",
+		"style-src 'self' " + cspSchemes + " 'unsafe-inline'",
+	}
+	if !cfg.IsDev {
+		cspDirectives = append(cspDirectives, "upgrade-insecure-requests")
+	}
+	csp := strings.Join(cspDirectives, ";")
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			h := c.Response().Header()
+			h.Set(echo.HeaderContentSecurityPolicy, csp)
+			h.Set("Cross-Origin-Opener-Policy", "same-origin")
+			h.Set("Cross-Origin-Resource-Policy", "same-origin")
+			h.Set(echo.HeaderReferrerPolicy, "no-referrer")
+			if cfg.HstsMaxAge != 0 {
+				h.Set(
+					echo.HeaderStrictTransportSecurity,
+					"max-age="+strconv.FormatInt(int64(cfg.HstsMaxAge), 10),
+				)
+			}
+			return next(c)
+		}
+	}
+}
+
+func getReqInfoFromHTTPHeaders(r *http.Request) (*app.ReqInfo, error) {
+	// header names must be in canonical form (see http.CanonicalHeaderKey)
+	usernames := r.Header["X-Csc-Adfs-Username"]
+	if len(usernames) == 0 {
+		return nil, errors.New("Username is missing from HTTP headers")
+	}
+	givenNames := r.Header["X-Csc-Adfs-Firstname"]
+	if len(givenNames) == 0 {
+		return nil, errors.New("Given name is missing from HTTP headers")
+	}
+	return &app.ReqInfo{Username: usernames[0], GivenName: givenNames[0]}, nil
+}
+
+type appContext struct {
+	echo.Context
+	req *app.ReqInfo
+	app *app.App
+}
+
+func (ac *appContext) Log() logging.Logger {
+	return ac.Context.Logger()
+}
+
+func (ac *appContext) Req() *app.ReqInfo {
+	return ac.req
+}
+
+func appContextMiddleware(app *app.App) echo.MiddlewareFunc {
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			reqInfo, err := getReqInfoFromHTTPHeaders(c.Request())
+			if err != nil {
+				return err
+			}
+			ac := &appContext{Context: c, req: reqInfo, app: app}
+			return next(ac)
+		}
+	}
+}