From 7e4b6a018a2638e8d867eead7bce0cc1edc01192 Mon Sep 17 00:00:00 2001 From: Max Erenberg <> Date: Thu, 9 Sep 2021 20:13:39 -0400 Subject: [PATCH 01/10] add docker.sh --- .drone.yml | 2 - .drone/auth1-setup.sh | 8 ++++ .drone/coffee-setup.sh | 2 + .drone/data.ldif | 31 +++++++++++++ .drone/mail-setup.sh | 14 ++++++ .drone/phosphoric-acid-setup.sh | 14 ++++++ .drone/uwldap_data.ldif | 78 +++++++++++++++++++++++++++++++++ docker.sh | 40 +++++++++++++++++ tests/MockMailmanServer.py | 10 ++++- tests/MockSMTPServer.py | 8 ++++ tests/ceod_dev.ini | 6 +-- 11 files changed, 206 insertions(+), 7 deletions(-) create mode 100755 .drone/mail-setup.sh create mode 100644 .drone/uwldap_data.ldif create mode 100755 docker.sh diff --git a/.drone.yml b/.drone.yml index 0dee1cc..1f6a0ab 100644 --- a/.drone.yml +++ b/.drone.yml @@ -28,12 +28,10 @@ services: image: debian:buster commands: - .drone/auth1-setup.sh - - sleep infinity - name: coffee image: debian:buster commands: - .drone/coffee-setup.sh - - sleep infinity trigger: branch: diff --git a/.drone/auth1-setup.sh b/.drone/auth1-setup.sh index 115d382..4ef21d1 100755 --- a/.drone/auth1-setup.sh +++ b/.drone/auth1-setup.sh @@ -18,6 +18,9 @@ export DEBIAN_FRONTEND=noninteractive apt update apt install -y psmisc +# If we don't do this then OpenLDAP uses a lot of RAM +ulimit -n 1024 + # LDAP apt install -y --no-install-recommends slapd ldap-utils libnss-ldapd sudo-ldap # `service slapd stop` doesn't seem to work @@ -40,6 +43,9 @@ sed -E -i 's/^base .*$/base dc=csclub,dc=internal/' /etc/nslcd.conf cp .drone/nsswitch.conf /etc/nsswitch.conf service nslcd start ldapadd -c -f .drone/data.ldif -Y EXTERNAL -H ldapi:/// +if [ -z "$CI" ]; then + ldapadd -c -f .drone/uwldap_data.ldif -Y EXTERNAL -H ldapi:/// +fi # KERBEROS apt install -y krb5-admin-server krb5-user libpam-krb5 libsasl2-modules-gssapi-mit sasl2-bin @@ -81,3 +87,5 @@ killall slapd && sleep 0.5 && service slapd start # sync with phosphoric-acid apt install -y netcat-openbsd nc -l 0.0.0.0 9000 + +sleep infinity diff --git a/.drone/coffee-setup.sh b/.drone/coffee-setup.sh index e84d137..53054f6 100755 --- a/.drone/coffee-setup.sh +++ b/.drone/coffee-setup.sh @@ -46,3 +46,5 @@ EOF" postgres # sync with phosphoric-acid apt install -y netcat-openbsd nc -l 0.0.0.0 9000 + +sleep infinity diff --git a/.drone/data.ldif b/.drone/data.ldif index 8b57ee7..6a50745 100644 --- a/.drone/data.ldif +++ b/.drone/data.ldif @@ -123,3 +123,34 @@ objectClass: group objectClass: posixGroup cn: regular1 gidNumber: 20002 + +dn: uid=exec1,ou=People,dc=csclub,dc=internal +cn: Regular One +userPassword: {SASL}exec1@CSCLUB.INTERNAL +loginShell: /bin/bash +homeDirectory: /users/exec1 +uid: exec1 +uidNumber: 20003 +gidNumber: 20003 +objectClass: top +objectClass: account +objectClass: posixAccount +objectClass: shadowAccount +objectClass: member +program: MAT/Mathematics Computer Science +term: s2021 + +dn: cn=exec1,ou=Group,dc=csclub,dc=internal +objectClass: top +objectClass: group +objectClass: posixGroup +cn: exec1 +gidNumber: 20003 + +dn: cn=exec,ou=Group,dc=csclub,dc=internal +objectClass: top +objectClass: group +objectClass: posixGroup +cn: syscom +gidNumber: 10013 +uniqueMember: uid=exec1,ou=People,dc=csclub,dc=internal diff --git a/.drone/mail-setup.sh b/.drone/mail-setup.sh new file mode 100755 index 0000000..527698a --- /dev/null +++ b/.drone/mail-setup.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +. .drone/common.sh + +# set FQDN in /etc/hosts +add_fqdn_to_hosts $(get_ip_addr $(hostname)) mail + +. venv/bin/activate +python tests/MockMailmanServer.py & +python tests/MockSMTPServer.py & + +sleep infinity diff --git a/.drone/phosphoric-acid-setup.sh b/.drone/phosphoric-acid-setup.sh index 1977dcd..e90d265 100755 --- a/.drone/phosphoric-acid-setup.sh +++ b/.drone/phosphoric-acid-setup.sh @@ -22,6 +22,10 @@ sync_with() { add_fqdn_to_hosts $(get_ip_addr $(hostname)) phosphoric-acid add_fqdn_to_hosts $(get_ip_addr auth1) auth1 add_fqdn_to_hosts $(get_ip_addr coffee) coffee +# mail container doesn't run in CI +if [ -z "$CI" ]; then + add_fqdn_to_hosts $(get_ip_addr mail) mail +fi export DEBIAN_FRONTEND=noninteractive apt update @@ -62,3 +66,13 @@ sync_with coffee shopt -s dotglob mkdir -p /users/skel cp /etc/skel/* /users/skel/ + +# create directories for users +for user in ctdalek regular1 exec1; do + mkdir /users/$user + chown $user:$user /users/$user +done + +if [ -z "$CI" ]; then + sleep infinity +fi diff --git a/.drone/uwldap_data.ldif b/.drone/uwldap_data.ldif new file mode 100644 index 0000000..4f9cf07 --- /dev/null +++ b/.drone/uwldap_data.ldif @@ -0,0 +1,78 @@ +dn: ou=People,dc=uwaterloo,dc=internal +objectClass: organizationalUnit +ou: People + +dn: uid=ctdalek,ou=People,dc=uwaterloo,dc=internal +displayName: Calum Dalek +givenName: Calum +sn: Dalek +cn: Calum Dalek +ou: MAT/Mathematics Computer Science +mailLocalAddress: ctdalek@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: ctdalek +mail: ctdalek@uwaterloo.internal + +dn: uid=regular1,ou=People,dc=uwaterloo,dc=internal +displayName: Regular One +givenName: Regular +sn: One +cn: Regular One +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular1 +mail: regular1@uwaterloo.internal + +dn: uid=regular2,ou=People,dc=uwaterloo,dc=internal +displayName: Regular Two +givenName: Regular +sn: Two +cn: Regular Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular2 +mail: regular2@uwaterloo.internal + +dn: uid=exec1,ou=People,dc=uwaterloo,dc=internal +displayName: Exec One +givenName: Exec +sn: One +cn: Exec One +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec1 +mail: exec1@uwaterloo.internal + +dn: uid=exec2,ou=People,dc=uwaterloo,dc=internal +displayName: Exec Two +givenName: Exec +sn: One +cn: Exec Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec2 +mail: exec2@uwaterloo.internal diff --git a/docker.sh b/docker.sh new file mode 100755 index 0000000..96e5301 --- /dev/null +++ b/docker.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -x + +if ! [ -d venv ]; then + docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster \ + sh -c "python -m venv && . venv/bin/activate && pip install -r dev-requirements.txt && pip install -r requirements.txt" +fi + +case $1 in + up) + docker network create ceod + for host in auth1 coffee mail phosphoric-acid; do + if [ $host = auth1 -o $host = coffee ]; then + image=debian:buster + else + image=python:3.7-buster + fi + docker run \ + --detach \ + --name $host \ + --hostname $host \ + --network ceod \ + --volume "$PWD:$PWD" \ + --workdir "$PWD" \ + $image .drone/$host-setup.sh + done + ;; + down) + for host in auth1 coffee mail phosphoric-acid; do + docker kill $host + docker rm $host + done + docker network rm ceod + ;; + *) + echo 'Usage: docker.sh ' >&2 + exit 1 + ;; +esac diff --git a/tests/MockMailmanServer.py b/tests/MockMailmanServer.py index 561d1ff..d2a3f02 100644 --- a/tests/MockMailmanServer.py +++ b/tests/MockMailmanServer.py @@ -4,7 +4,8 @@ from aiohttp import web class MockMailmanServer: - def __init__(self): + def __init__(self, port=8002): + self.port = port self.app = web.Application() self.app.add_routes([ web.post('/members', self.subscribe), @@ -24,7 +25,7 @@ class MockMailmanServer: def _start_loop(self): asyncio.set_event_loop(self.loop) self.loop.run_until_complete(self.runner.setup()) - site = web.TCPSite(self.runner, '127.0.0.1', 8002) + site = web.TCPSite(self.runner, '127.0.0.1', self.port) self.loop.run_until_complete(site.start()) self.loop.run_forever() @@ -67,3 +68,8 @@ class MockMailmanServer: }, status=404) subscribers.remove(subscriber) return web.json_response({'status': 'OK'}) + + +if __name__ == '__main__': + server = MockMailmanServer(8001) + server.start() diff --git a/tests/MockSMTPServer.py b/tests/MockSMTPServer.py index 0a70c1f..ebcc750 100644 --- a/tests/MockSMTPServer.py +++ b/tests/MockSMTPServer.py @@ -1,3 +1,5 @@ +import os + from aiosmtpd.controller import Controller @@ -25,3 +27,9 @@ class MockHandler: } self.mock_server.messages.append(msg) return '250 Message accepted for delivery' + + +if __name__ == '__main__': + assert os.geteuid() == 0 + server = MockSMTPServer('0.0.0.0', 25) + server.start() diff --git a/tests/ceod_dev.ini b/tests/ceod_dev.ini index f577120..d73955f 100644 --- a/tests/ceod_dev.ini +++ b/tests/ceod_dev.ini @@ -13,15 +13,15 @@ port = 9987 [ldap] admin_principal = ceod/admin -server_url = ldap://ldap-master.csclub.internal +server_url = ldap://auth1.csclub.internal sasl_realm = CSCLUB.INTERNAL users_base = ou=People,dc=csclub,dc=internal groups_base = ou=Group,dc=csclub,dc=internal sudo_base = ou=SUDOers,dc=csclub,dc=internal [uwldap] -server_url = ldap://uwldap.uwaterloo.ca -base = dc=uwaterloo,dc=ca +server_url = ldap://auth1.csclub.internal +base = dc=uwaterloo,dc=internal [members] min_id = 20001 -- 2.39.2 From 6862ff4a632e028fbfbbc18795ba205929177bdc Mon Sep 17 00:00:00 2001 From: Max Erenberg <> Date: Thu, 9 Sep 2021 23:21:22 -0400 Subject: [PATCH 02/10] add principal for mail --- .drone/auth1-setup.sh | 4 ++++ .drone/common.sh | 18 ++++++++++++++++++ .drone/data.ldif | 2 +- .drone/mail-setup.sh | 22 ++++++++++++++++++++++ .drone/phosphoric-acid-setup.sh | 17 +++-------------- .drone/uwldap_data.ldif | 12 ++++++------ docker.sh | 9 ++++----- tests/MockSMTPServer.py | 2 ++ tests/ceod_dev.ini | 2 +- 9 files changed, 61 insertions(+), 27 deletions(-) diff --git a/.drone/auth1-setup.sh b/.drone/auth1-setup.sh index 4ef21d1..c8486e8 100755 --- a/.drone/auth1-setup.sh +++ b/.drone/auth1-setup.sh @@ -87,5 +87,9 @@ killall slapd && sleep 0.5 && service slapd start # sync with phosphoric-acid apt install -y netcat-openbsd nc -l 0.0.0.0 9000 +if [ -z "$CI" ]; then + # sync with mail + nc -l 0.0.0.0 9001 +fi sleep infinity diff --git a/.drone/common.sh b/.drone/common.sh index ed91d08..740fc93 100644 --- a/.drone/common.sh +++ b/.drone/common.sh @@ -15,3 +15,21 @@ add_fqdn_to_hosts() { rm /tmp/hosts echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts } + +sync_with() { + host=$1 + port=9000 + if [ $# -eq 2 ]; then + port=$2 + fi + synced=false + # give it 5 minutes + for i in {1..60}; do + if nc -vz $host $port ; then + synced=true + break + fi + sleep 5 + done + test $synced = true +} diff --git a/.drone/data.ldif b/.drone/data.ldif index 6a50745..48873b1 100644 --- a/.drone/data.ldif +++ b/.drone/data.ldif @@ -151,6 +151,6 @@ dn: cn=exec,ou=Group,dc=csclub,dc=internal objectClass: top objectClass: group objectClass: posixGroup -cn: syscom +cn: exec gidNumber: 10013 uniqueMember: uid=exec1,ou=People,dc=csclub,dc=internal diff --git a/.drone/mail-setup.sh b/.drone/mail-setup.sh index 527698a..42745b2 100755 --- a/.drone/mail-setup.sh +++ b/.drone/mail-setup.sh @@ -6,9 +6,31 @@ set -ex # set FQDN in /etc/hosts add_fqdn_to_hosts $(get_ip_addr $(hostname)) mail +add_fqdn_to_hosts $(get_ip_addr auth1) auth1 . venv/bin/activate python tests/MockMailmanServer.py & python tests/MockSMTPServer.py & +# KERBEROS +export DEBIAN_FRONTEND=noninteractive +apt update +apt install -y krb5-user netcat-openbsd +cp .drone/krb5.conf /etc/krb5.conf + +# sync with auth1 +sync_with auth1 9001 + +rm -f /etc/krb5.keytab +cat < Date: Thu, 9 Sep 2021 23:37:53 -0400 Subject: [PATCH 03/10] update README --- README.md | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2a8e1b9..ab010a5 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,35 @@ club accounts and memberships. See [architecture.md](architecture.md) for an overview of its architecture. ## Development -First, make sure that you have installed the +### Docker +If you are not modifying code related to email or Mailman, then you may use +Docker containers instead, which are much easier to work with than the VM. +``` +docker.sh up +``` +This will create some containers with the bare minimum necessary for ceod to +run. Run `docker logs -f phosphoric-acid` and wait until you see the line +`sleep infinity`. Then attach to each of phosphoric-acid, mail and coffee, +and start ceod (see 'Running the application', below). Once inside a container, +make sure to `cd` into the current working directory on the host. + +To use ceo, run the following inside the phosphoric-acid container: +``` +login + + +. venv/bin/activate +python -m ceo +``` +This should bring up the TUI. + +### VM +If you are making changes related to email or Mailman, you will need the full [syscom dev environment](https://git.uwaterloo.ca/csc/syscom-dev-environment). This will setup all of the services needed for ceo to work. You should clone this repo in the phosphoric-acid container under ctdalek's home directory; you will then be able to access it from any container thanks to NFS. -### Environment setup Once you have the dev environment setup, there are a few more steps you'll need to do for ceo. @@ -148,8 +170,16 @@ is running. Stop the flask app (Ctrl-C), run `clear_cache.sh`, then restart the app. ## Interacting with the application -The client part of ceo hasn't been written yet, so we'll use curl to -interact with ceod for now. +To use the TUI: +``` +python -m ceo +``` +To use the CLI: +``` +python -m ceo --help +``` + +Alternatively, you may use curl to send HTTP requests. ceod uses [SPNEGO](https://en.wikipedia.org/wiki/SPNEGO) for authentication, and TLS for confidentiality and integrity. In development mode, TLS can be -- 2.39.2 From f72c6665f902dbde05f563fa317051154aa33561 Mon Sep 17 00:00:00 2001 From: Max Erenberg <> Date: Fri, 10 Sep 2021 18:29:32 -0400 Subject: [PATCH 04/10] add nslcd for mail and coffee --- .drone/auth1-setup.sh | 15 +++++++++--- .drone/coffee-setup.sh | 7 +++++- .drone/common.sh | 42 +++++++++++++++++++++++++++++++++ .drone/mail-setup.sh | 17 ++----------- .drone/phosphoric-acid-setup.sh | 37 +++-------------------------- 5 files changed, 65 insertions(+), 53 deletions(-) diff --git a/.drone/auth1-setup.sh b/.drone/auth1-setup.sh index c8486e8..5364618 100755 --- a/.drone/auth1-setup.sh +++ b/.drone/auth1-setup.sh @@ -64,6 +64,7 @@ cat < $POSTGRES_DIR/pg_hba.conf @@ -43,8 +46,10 @@ REVOKE ALL ON SCHEMA public FROM public; GRANT ALL ON SCHEMA public TO postgres; EOF" postgres -# sync with phosphoric-acid apt install -y netcat-openbsd +auth_setup coffee + +# sync with phosphoric-acid nc -l 0.0.0.0 9000 sleep infinity diff --git a/.drone/common.sh b/.drone/common.sh index 740fc93..0f7c287 100644 --- a/.drone/common.sh +++ b/.drone/common.sh @@ -33,3 +33,45 @@ sync_with() { done test $synced = true } + +auth_setup() { + hostname=$1 + + # LDAP + apt install -y --no-install-recommends libnss-ldapd + service nslcd stop || true + cp .drone/ldap.conf /etc/ldap/ldap.conf + grep -Eq '^map group member uniqueMember$' /etc/nslcd.conf || \ + echo 'map group member uniqueMember' >> /etc/nslcd.conf + sed -E -i 's/^uri .*$/uri ldap:\/\/auth1.csclub.internal/' /etc/nslcd.conf + sed -E -i 's/^base .*$/base dc=csclub,dc=internal/' /etc/nslcd.conf + cp .drone/nsswitch.conf /etc/nsswitch.conf + + # KERBEROS + apt install -y krb5-user libpam-krb5 libsasl2-modules-gssapi-mit + cp .drone/krb5.conf /etc/krb5.conf + + if [ $hostname = phosphoric-acid ]; then + sync_port=9000 + elif [ $hostname = coffee ]; then + sync_port=9001 + else + sync_port=9002 + fi + sync_with auth1 $sync_port + + rm -f /etc/krb5.keytab + cat <> /etc/nslcd.conf -sed -E -i 's/^uri .*$/uri ldap:\/\/auth1.csclub.internal/' /etc/nslcd.conf -sed -E -i 's/^base .*$/base dc=csclub,dc=internal/' /etc/nslcd.conf -cp .drone/nsswitch.conf /etc/nsswitch.conf - -# KERBEROS -apt install -y krb5-user libpam-krb5 libsasl2-modules-gssapi-mit -cp .drone/krb5.conf /etc/krb5.conf - apt install -y netcat-openbsd - -sync_with auth1 - -rm -f /etc/krb5.keytab -cat < Date: Fri, 10 Sep 2021 18:45:53 -0400 Subject: [PATCH 05/10] don't setup auth on coffee in CI --- .drone/coffee-setup.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.drone/coffee-setup.sh b/.drone/coffee-setup.sh index ac8822a..454fc2c 100755 --- a/.drone/coffee-setup.sh +++ b/.drone/coffee-setup.sh @@ -47,7 +47,9 @@ GRANT ALL ON SCHEMA public TO postgres; EOF" postgres apt install -y netcat-openbsd -auth_setup coffee +if [ -z "$CI" ]; then + auth_setup coffee +fi # sync with phosphoric-acid nc -l 0.0.0.0 9000 -- 2.39.2 From dca957d61a7a57f245a1bbdbe545b401abef27b3 Mon Sep 17 00:00:00 2001 From: Max Erenberg <> Date: Fri, 10 Sep 2021 18:51:27 -0400 Subject: [PATCH 06/10] update OU --- .drone/uwldap_data.ldif | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone/uwldap_data.ldif b/.drone/uwldap_data.ldif index 2ba826d..51491bd 100644 --- a/.drone/uwldap_data.ldif +++ b/.drone/uwldap_data.ldif @@ -1,6 +1,6 @@ dn: ou=UWLDAP,dc=csclub,dc=internal objectClass: organizationalUnit -ou: People +ou: UWLDAP dn: uid=ctdalek,ou=UWLDAP,dc=csclub,dc=internal displayName: Calum Dalek -- 2.39.2 From ae34b6b7ce330ffab9eab020713ed44223da517b Mon Sep 17 00:00:00 2001 From: Max Erenberg <> Date: Fri, 10 Sep 2021 22:41:07 -0400 Subject: [PATCH 07/10] add more UWLDAP users --- .drone/auth1-setup.sh | 14 +-- .drone/uwldap_data.ldif | 227 +++++++++++++++++++++++++++++++++++++++- docker.sh | 8 +- 3 files changed, 238 insertions(+), 11 deletions(-) diff --git a/.drone/auth1-setup.sh b/.drone/auth1-setup.sh index 5364618..57baa61 100755 --- a/.drone/auth1-setup.sh +++ b/.drone/auth1-setup.sh @@ -7,12 +7,14 @@ set -ex # set FQDN in /etc/hosts add_fqdn_to_hosts $(get_ip_addr $(hostname)) auth1 -# I'm not sure why, but we also need to remove the hosts entry for the -# container's real hostname, otherwise slapd only looks for the principal -# ldap/ (this is with the sasl-host option) -sed -E "/\\b$(hostname)\\b/d" /etc/hosts > /tmp/hosts -cat /tmp/hosts > /etc/hosts -rm /tmp/hosts +if [ -n "$CI" ]; then + # I'm not sure why, but we also need to remove the hosts entry for the + # container's real hostname, otherwise slapd only looks for the principal + # ldap/ (this is with the sasl-host option) + sed -E "/\\b$(hostname)\\b/d" /etc/hosts > /tmp/hosts + cat /tmp/hosts > /etc/hosts + rm /tmp/hosts +fi export DEBIAN_FRONTEND=noninteractive apt update diff --git a/.drone/uwldap_data.ldif b/.drone/uwldap_data.ldif index 51491bd..89d2d66 100644 --- a/.drone/uwldap_data.ldif +++ b/.drone/uwldap_data.ldif @@ -65,7 +65,7 @@ mail: exec1@uwaterloo.internal dn: uid=exec2,ou=UWLDAP,dc=csclub,dc=internal displayName: Exec Two givenName: Exec -sn: One +sn: Two cn: Exec Two ou: MAT/Mathematics Computer Science mailLocalAddress: exec2@uwaterloo.internal @@ -76,3 +76,228 @@ objectClass: person objectClass: top uid: exec2 mail: exec2@uwaterloo.internal + +dn: uid=ctdalek,ou=UWLDAP,dc=csclub,dc=internal +displayName: Calum Dalek +givenName: Calum +sn: Dalek +cn: Calum Dalek +ou: MAT/Mathematics Computer Science +mailLocalAddress: ctdalek@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: ctdalek +mail: ctdalek@uwaterloo.internal + +dn: uid=regular1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular One +givenName: Regular +sn: One +cn: Regular One +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular1 +mail: regular1@uwaterloo.internal + +dn: uid=regular2,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular Two +givenName: Regular +sn: Two +cn: Regular Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular2 +mail: regular2@uwaterloo.internal + +dn: uid=exec1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec One +givenName: Exec +sn: One +cn: Exec One +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec1 +mail: exec1@uwaterloo.internal + +dn: uid=ctdalek,ou=UWLDAP,dc=csclub,dc=internal +displayName: Calum Dalek +givenName: Calum +sn: Dalek +cn: Calum Dalek +ou: MAT/Mathematics Computer Science +mailLocalAddress: ctdalek@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: ctdalek +mail: ctdalek@uwaterloo.internal + +dn: uid=regular1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular One +givenName: Regular +sn: One +cn: Regular One +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular1 +mail: regular1@uwaterloo.internal + +dn: uid=regular2,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular Two +givenName: Regular +sn: Two +cn: Regular Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular2 +mail: regular2@uwaterloo.internal + +dn: uid=exec1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec One +givenName: Exec +sn: One +cn: Exec One +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec1 +mail: exec1@uwaterloo.internal + +dn: uid=ctdalek,ou=UWLDAP,dc=csclub,dc=internal +displayName: Calum Dalek +givenName: Calum +sn: Dalek +cn: Calum Dalek +ou: MAT/Mathematics Computer Science +mailLocalAddress: ctdalek@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: ctdalek +mail: ctdalek@uwaterloo.internal + +dn: uid=regular1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular One +givenName: Regular +sn: One +cn: Regular One +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular1 +mail: regular1@uwaterloo.internal + +dn: uid=regular2,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular Two +givenName: Regular +sn: Two +cn: Regular Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular2 +mail: regular2@uwaterloo.internal + +dn: uid=regular3,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular Three +givenName: Regular +sn: Three +cn: Regular Three +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular3@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular3 +mail: regular3@uwaterloo.internal + +dn: uid=exec1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec One +givenName: Exec +sn: One +cn: Exec One +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec1 +mail: exec1@uwaterloo.internal + +dn: uid=exec2,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec Two +givenName: Exec +sn: Two +cn: Exec Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec2 +mail: exec2@uwaterloo.internal + +dn: uid=exec3,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec Three +givenName: Exec +sn: Three +cn: Exec Three +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec3@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec3 +mail: exec3@uwaterloo.internal diff --git a/docker.sh b/docker.sh index dcb9441..d5ebac1 100755 --- a/docker.sh +++ b/docker.sh @@ -4,10 +4,10 @@ set -x case $1 in up) - if ! [ -d venv ]; then - docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster \ - sh -c "python -m venv venv && . venv/bin/activate && pip install -r dev-requirements.txt && pip install -r requirements.txt" - fi + if ! [ -d venv ]; then + docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster \ + sh -c "python -m venv venv && . venv/bin/activate && pip install -r dev-requirements.txt && pip install -r requirements.txt" + fi docker network create ceod for host in auth1 coffee mail phosphoric-acid; do if [ $host = auth1 -o $host = coffee ]; then -- 2.39.2 From 35e577b86611f2e8bf9542b3c8c21e1a84e0294f Mon Sep 17 00:00:00 2001 From: Max Erenberg <> Date: Fri, 10 Sep 2021 22:57:06 -0400 Subject: [PATCH 08/10] use python image for coffee --- docker.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker.sh b/docker.sh index d5ebac1..9c6fdf9 100755 --- a/docker.sh +++ b/docker.sh @@ -10,7 +10,7 @@ case $1 in fi docker network create ceod for host in auth1 coffee mail phosphoric-acid; do - if [ $host = auth1 -o $host = coffee ]; then + if [ $host = auth1 ]; then image=debian:buster else image=python:3.7-buster @@ -22,6 +22,8 @@ case $1 in --network ceod \ --volume "$PWD:$PWD" \ --workdir "$PWD" \ + --env FLASK_APP=ceod.api \ + --env FLASK_ENV=development \ $image .drone/$host-setup.sh done ;; -- 2.39.2 From f873d0413955cd34a63206b3804efcf5c4d21b70 Mon Sep 17 00:00:00 2001 From: Rio Date: Wed, 15 Sep 2021 17:30:49 -0400 Subject: [PATCH 09/10] Use docker-compose to manage containers. (#19) Use docker-compose to manage the containers, and start ceod automatically. The service can be restarted with SIGHUP sent by docker/docker-compose. Co-authored-by: Rio6 Reviewed-on: https://git.csclub.uwaterloo.ca/public/pyceo/pulls/19 Co-authored-by: Rio Co-committed-by: Rio --- .drone.yml | 2 ++ .drone/auth1-setup.sh | 10 +++----- .drone/coffee-setup.sh | 6 ++--- .drone/mail-setup.sh | 6 ++--- .drone/phosphoric-acid-setup.sh | 9 +++---- .drone/supervise.sh | 17 +++++++++++++ README.md | 37 +++++++++++++++++---------- docker-compose.yml | 44 +++++++++++++++++++++++++++++++++ docker-entrypoint.sh | 10 ++++++++ docker.sh | 41 ------------------------------ 10 files changed, 109 insertions(+), 73 deletions(-) create mode 100755 .drone/supervise.sh create mode 100644 docker-compose.yml create mode 100755 docker-entrypoint.sh delete mode 100755 docker.sh diff --git a/.drone.yml b/.drone.yml index 1f6a0ab..0dee1cc 100644 --- a/.drone.yml +++ b/.drone.yml @@ -28,10 +28,12 @@ services: image: debian:buster commands: - .drone/auth1-setup.sh + - sleep infinity - name: coffee image: debian:buster commands: - .drone/coffee-setup.sh + - sleep infinity trigger: branch: diff --git a/.drone/auth1-setup.sh b/.drone/auth1-setup.sh index 57baa61..a96d017 100755 --- a/.drone/auth1-setup.sh +++ b/.drone/auth1-setup.sh @@ -46,7 +46,7 @@ cp .drone/nsswitch.conf /etc/nsswitch.conf service nslcd start ldapadd -c -f .drone/data.ldif -Y EXTERNAL -H ldapi:/// if [ -z "$CI" ]; then - ldapadd -c -f .drone/uwldap_data.ldif -Y EXTERNAL -H ldapi:/// + ldapadd -c -f .drone/uwldap_data.ldif -Y EXTERNAL -H ldapi:/// ||: fi # KERBEROS @@ -95,12 +95,10 @@ done apt install -y netcat-openbsd # sync with phosphoric-acid -nc -l 0.0.0.0 9000 +nc -l 0.0.0.0 9000 & if [ -z "$CI" ]; then # sync with coffee - nc -l 0.0.0.0 9001 + nc -l 0.0.0.0 9001 & # sync with mail - nc -l 0.0.0.0 9002 + nc -l 0.0.0.0 9002 & fi - -sleep infinity diff --git a/.drone/coffee-setup.sh b/.drone/coffee-setup.sh index 454fc2c..02ecc55 100755 --- a/.drone/coffee-setup.sh +++ b/.drone/coffee-setup.sh @@ -18,7 +18,7 @@ service mysql stop sed -E -i 's/^(bind-address[[:space:]]+= 127.0.0.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf service mysql start cat </dev/null' HUP +trap 'running=0; kill -TERM $! 2>/dev/null' TERM INT +trap 'running=0; kill -KILL $! 2>/dev/null' EXIT + +while [ "$running" = 1 ]; do + "$@" & + wait + sleep "$TIMEOUT" +done diff --git a/README.md b/README.md index ab010a5..771f798 100644 --- a/README.md +++ b/README.md @@ -10,26 +10,37 @@ overview of its architecture. If you are not modifying code related to email or Mailman, then you may use Docker containers instead, which are much easier to work with than the VM. ``` -docker.sh up +docker-compose up -d # or without -d to run in the foreground ``` This will create some containers with the bare minimum necessary for ceod to -run. Run `docker logs -f phosphoric-acid` and wait until you see the line -`sleep infinity`. Then attach to each of phosphoric-acid, mail and coffee, -and start ceod (see 'Running the application', below). Once inside a container, -make sure to `cd` into the current working directory on the host. - -To use ceo, run the following inside the phosphoric-acid container: +run, and start ceod on each of phosphoric-acid, mail, and coffee container. +You can check the containers status using: +```sh +docker-compose logs -f ``` -login - - -. venv/bin/activate -python -m ceo + +To use ceo, run the following: +```sh +docker-compose exec phosphoric-acid bash +su ctdalek +python -m ceo # the password for kerobos is krb5 ``` This should bring up the TUI. +Normally, ceod should autoamtically restart when the source files are changed. +To manually restart the service, run: +```sh +docker-compose kill -s SIGHUP phosphoric-acid +``` + +To stop the containers, run: +```sh +docker-compose down +``` +Alternatively, if you started docker-compose in the foreground, just press Ctrl-C. + ### VM -If you are making changes related to email or Mailman, you will need the full +If you need the full environment running in VM, follow the guide on [syscom dev environment](https://git.uwaterloo.ca/csc/syscom-dev-environment). This will setup all of the services needed for ceo to work. You should clone this repo in the phosphoric-acid container under ctdalek's home directory; you diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7c4ea6f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,44 @@ +version: "3.6" + +x-common: &common + image: python:3.7-buster + volumes: + - .:$PWD + environment: + FLASK_APP: ceod.api + FLASK_ENV: development + working_dir: $PWD + entrypoint: + - ./docker-entrypoint.sh + +services: + auth1: + <<: *common + image: debian:buster + hostname: auth0 + command: auth1 + + coffee: + <<: *common + command: coffee + hostname: coffee + depends_on: + - auth1 + + mail: + <<: *common + command: mail + hostname: mail + depends_on: + - auth1 + + phosphoric-acid: + <<: *common + command: phosphoric-acid + hostname: phosphoric-acid + depends_on: + - auth1 + - coffee + - mail + +# vim: expandtab sw=2 ts=2 diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..8d430b2 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/sh -e +host="$1" +[ -x ".drone/$host-setup.sh" ] && "./.drone/$host-setup.sh" + +if [ "$host" = auth1 ]; then + exec sleep infinity +else + python3 -m pip install -r requirements.txt -r dev-requirements.txt + exec ./.drone/supervise.sh flask run -h 0.0.0.0 -p 9987 +fi diff --git a/docker.sh b/docker.sh deleted file mode 100755 index 9c6fdf9..0000000 --- a/docker.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -x - -case $1 in - up) - if ! [ -d venv ]; then - docker run --rm -v "$PWD:$PWD" -w "$PWD" python:3.7-buster \ - sh -c "python -m venv venv && . venv/bin/activate && pip install -r dev-requirements.txt && pip install -r requirements.txt" - fi - docker network create ceod - for host in auth1 coffee mail phosphoric-acid; do - if [ $host = auth1 ]; then - image=debian:buster - else - image=python:3.7-buster - fi - docker run \ - --detach \ - --name $host \ - --hostname $host \ - --network ceod \ - --volume "$PWD:$PWD" \ - --workdir "$PWD" \ - --env FLASK_APP=ceod.api \ - --env FLASK_ENV=development \ - $image .drone/$host-setup.sh - done - ;; - down) - for host in auth1 coffee mail phosphoric-acid; do - docker kill $host - docker rm $host - done - docker network rm ceod - ;; - *) - echo 'Usage: docker.sh ' >&2 - exit 1 - ;; -esac -- 2.39.2 From d7e9de5f61b67c6d4307314aad3d6548f1638e5c Mon Sep 17 00:00:00 2001 From: Max Erenberg <> Date: Fri, 17 Sep 2021 22:29:34 -0400 Subject: [PATCH 10/10] install virtualenv on host --- .drone/auth1-setup.sh | 6 +++++- README.md | 14 +++++++++++--- docker-compose.yml | 2 +- docker-entrypoint.sh | 12 +++++++++--- tests/MockMailmanServer.py | 8 ++++---- tests/ceod_test_local.ini | 4 ++-- 6 files changed, 32 insertions(+), 14 deletions(-) diff --git a/.drone/auth1-setup.sh b/.drone/auth1-setup.sh index a96d017..53f52f2 100755 --- a/.drone/auth1-setup.sh +++ b/.drone/auth1-setup.sh @@ -46,7 +46,11 @@ cp .drone/nsswitch.conf /etc/nsswitch.conf service nslcd start ldapadd -c -f .drone/data.ldif -Y EXTERNAL -H ldapi:/// if [ -z "$CI" ]; then - ldapadd -c -f .drone/uwldap_data.ldif -Y EXTERNAL -H ldapi:/// ||: + ldapadd -c -f .drone/uwldap_data.ldif -Y EXTERNAL -H ldapi:/// || true + # setup ldapvi for convenience + apt install -y vim ldapvi + echo 'export EDITOR=vim' >> /root/.bashrc + echo 'alias ldapvi="ldapvi -Y EXTERNAL -h ldapi:///"' >> /root/.bashrc fi # KERBEROS diff --git a/README.md b/README.md index 771f798..ef3e47d 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,15 @@ overview of its architecture. ### Docker If you are not modifying code related to email or Mailman, then you may use Docker containers instead, which are much easier to work with than the VM. + +First, make sure you create the virtualenv: +```sh +docker run --rm -v "$PWD:$PWD" -w "$PWD" -u $(id -u):$(id -g) python:3.7-buster \ + sh -c 'python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt' ``` -docker-compose up -d # or without -d to run in the foreground +Then bring up the containers: +```sh +docker-compose up -d # or without -d to run in the foreground ``` This will create some containers with the bare minimum necessary for ceod to run, and start ceod on each of phosphoric-acid, mail, and coffee container. @@ -23,7 +30,8 @@ To use ceo, run the following: ```sh docker-compose exec phosphoric-acid bash su ctdalek -python -m ceo # the password for kerobos is krb5 +. venv/bin/activate +python -m ceo # the password is krb5 ``` This should bring up the TUI. @@ -162,7 +170,7 @@ pip install -r requirements.txt pip install -r dev-requirements.txt ``` -## Running the application +#### Running the application ceod is a distributed application, with instances on different hosts offering different services. Therefore, you will need to run ceod on multiple hosts. Currently, those are diff --git a/docker-compose.yml b/docker-compose.yml index 7c4ea6f..324d324 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: auth1: <<: *common image: debian:buster - hostname: auth0 + hostname: auth1 command: auth1 coffee: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 8d430b2..9f19ecb 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,10 +1,16 @@ #!/bin/sh -e + +if ! [ -d venv ]; then + echo "You need to create the virtualenv first!" >&2 + exit 1 +fi + host="$1" -[ -x ".drone/$host-setup.sh" ] && "./.drone/$host-setup.sh" +[ -x ".drone/$host-setup.sh" ] && ".drone/$host-setup.sh" if [ "$host" = auth1 ]; then exec sleep infinity else - python3 -m pip install -r requirements.txt -r dev-requirements.txt - exec ./.drone/supervise.sh flask run -h 0.0.0.0 -p 9987 + . venv/bin/activate + exec .drone/supervise.sh flask run -h 0.0.0.0 -p 9987 fi diff --git a/tests/MockMailmanServer.py b/tests/MockMailmanServer.py index d2a3f02..d6ff78b 100644 --- a/tests/MockMailmanServer.py +++ b/tests/MockMailmanServer.py @@ -4,12 +4,12 @@ from aiohttp import web class MockMailmanServer: - def __init__(self, port=8002): + def __init__(self, port=8001, prefix='/3.1'): self.port = port self.app = web.Application() self.app.add_routes([ - web.post('/members', self.subscribe), - web.delete('/lists/{mailing_list}/member/{address}', self.unsubscribe), + web.post(prefix + '/members', self.subscribe), + web.delete(prefix + '/lists/{mailing_list}/member/{address}', self.unsubscribe), ]) self.runner = web.AppRunner(self.app) self.loop = asyncio.new_event_loop() @@ -71,5 +71,5 @@ class MockMailmanServer: if __name__ == '__main__': - server = MockMailmanServer(8001) + server = MockMailmanServer() server.start() diff --git a/tests/ceod_test_local.ini b/tests/ceod_test_local.ini index 25d9dae..7927654 100644 --- a/tests/ceod_test_local.ini +++ b/tests/ceod_test_local.ini @@ -9,7 +9,7 @@ fs_root_host = phosphoric-acid mailman_host = phosphoric-acid database_host = phosphoric-acid use_https = false -port = 9987 +port = 9988 [ldap] admin_principal = ceod/admin @@ -40,7 +40,7 @@ smtp_url = smtp://localhost:8025 smtp_starttls = false [mailman3] -api_base_url = http://localhost:8002 +api_base_url = http://localhost:8001/3.1 api_username = restadmin api_password = mailman3 new_member_list = csc-general -- 2.39.2