From de18a9f29397c497cb4b8a3c9eeeda20e3f6cfe9 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Fri, 17 Sep 2021 22:39:27 -0400 Subject: [PATCH] add option to use Docker instead of VM (#16) Co-authored-by: Max Erenberg <> Co-authored-by: Rio Reviewed-on: https://git.csclub.uwaterloo.ca/public/pyceo/pulls/16 Co-authored-by: Max Erenberg Co-committed-by: Max Erenberg --- .drone/auth1-setup.sh | 43 ++++- .drone/coffee-setup.sh | 13 +- .drone/common.sh | 60 +++++++ .drone/data.ldif | 31 ++++ .drone/mail-setup.sh | 21 +++ .drone/phosphoric-acid-setup.sh | 67 ++----- .drone/supervise.sh | 17 ++ .drone/uwldap_data.ldif | 303 ++++++++++++++++++++++++++++++++ README.md | 59 ++++++- docker-compose.yml | 44 +++++ docker-entrypoint.sh | 16 ++ tests/MockMailmanServer.py | 14 +- tests/MockSMTPServer.py | 10 ++ tests/ceod_dev.ini | 4 +- tests/ceod_test_local.ini | 4 +- 15 files changed, 633 insertions(+), 73 deletions(-) create mode 100755 .drone/mail-setup.sh create mode 100755 .drone/supervise.sh create mode 100644 .drone/uwldap_data.ldif create mode 100644 docker-compose.yml create mode 100755 docker-entrypoint.sh diff --git a/.drone/auth1-setup.sh b/.drone/auth1-setup.sh index 115d382..53f52f2 100755 --- a/.drone/auth1-setup.sh +++ b/.drone/auth1-setup.sh @@ -7,17 +7,22 @@ set -ex # set FQDN in /etc/hosts add_fqdn_to_hosts $(get_ip_addr $(hostname)) auth1 -# I'm not sure why, but we also need to remove the hosts entry for the -# container's real hostname, otherwise slapd only looks for the principal -# ldap/ (this is with the sasl-host option) -sed -E "/\\b$(hostname)\\b/d" /etc/hosts > /tmp/hosts -cat /tmp/hosts > /etc/hosts -rm /tmp/hosts +if [ -n "$CI" ]; then + # I'm not sure why, but we also need to remove the hosts entry for the + # container's real hostname, otherwise slapd only looks for the principal + # ldap/ (this is with the sasl-host option) + sed -E "/\\b$(hostname)\\b/d" /etc/hosts > /tmp/hosts + cat /tmp/hosts > /etc/hosts + rm /tmp/hosts +fi export DEBIAN_FRONTEND=noninteractive apt update apt install -y psmisc +# If we don't do this then OpenLDAP uses a lot of RAM +ulimit -n 1024 + # LDAP apt install -y --no-install-recommends slapd ldap-utils libnss-ldapd sudo-ldap # `service slapd stop` doesn't seem to work @@ -40,6 +45,13 @@ sed -E -i 's/^base .*$/base dc=csclub,dc=internal/' /etc/nslcd.conf cp .drone/nsswitch.conf /etc/nsswitch.conf service nslcd start ldapadd -c -f .drone/data.ldif -Y EXTERNAL -H ldapi:/// +if [ -z "$CI" ]; then + ldapadd -c -f .drone/uwldap_data.ldif -Y EXTERNAL -H ldapi:/// || true + # setup ldapvi for convenience + apt install -y vim ldapvi + echo 'export EDITOR=vim' >> /root/.bashrc + echo 'alias ldapvi="ldapvi -Y EXTERNAL -h ldapi:///"' >> /root/.bashrc +fi # KERBEROS apt install -y krb5-admin-server krb5-user libpam-krb5 libsasl2-modules-gssapi-mit sasl2-bin @@ -58,6 +70,7 @@ cat < $POSTGRES_DIR/pg_hba.conf @@ -43,6 +46,10 @@ REVOKE ALL ON SCHEMA public FROM public; GRANT ALL ON SCHEMA public TO postgres; EOF" postgres -# sync with phosphoric-acid apt install -y netcat-openbsd -nc -l 0.0.0.0 9000 +if [ -z "$CI" ]; then + auth_setup coffee +fi + +# sync with phosphoric-acid +nc -l 0.0.0.0 9000 & diff --git a/.drone/common.sh b/.drone/common.sh index ed91d08..0f7c287 100644 --- a/.drone/common.sh +++ b/.drone/common.sh @@ -15,3 +15,63 @@ add_fqdn_to_hosts() { rm /tmp/hosts echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts } + +sync_with() { + host=$1 + port=9000 + if [ $# -eq 2 ]; then + port=$2 + fi + synced=false + # give it 5 minutes + for i in {1..60}; do + if nc -vz $host $port ; then + synced=true + break + fi + sleep 5 + done + test $synced = true +} + +auth_setup() { + hostname=$1 + + # LDAP + apt install -y --no-install-recommends libnss-ldapd + service nslcd stop || true + cp .drone/ldap.conf /etc/ldap/ldap.conf + grep -Eq '^map group member uniqueMember$' /etc/nslcd.conf || \ + echo 'map group member uniqueMember' >> /etc/nslcd.conf + sed -E -i 's/^uri .*$/uri ldap:\/\/auth1.csclub.internal/' /etc/nslcd.conf + sed -E -i 's/^base .*$/base dc=csclub,dc=internal/' /etc/nslcd.conf + cp .drone/nsswitch.conf /etc/nsswitch.conf + + # KERBEROS + apt install -y krb5-user libpam-krb5 libsasl2-modules-gssapi-mit + cp .drone/krb5.conf /etc/krb5.conf + + if [ $hostname = phosphoric-acid ]; then + sync_port=9000 + elif [ $hostname = coffee ]; then + sync_port=9001 + else + sync_port=9002 + fi + sync_with auth1 $sync_port + + rm -f /etc/krb5.keytab + cat <> /etc/nslcd.conf -sed -E -i 's/^uri .*$/uri ldap:\/\/auth1.csclub.internal/' /etc/nslcd.conf -sed -E -i 's/^base .*$/base dc=csclub,dc=internal/' /etc/nslcd.conf -cp .drone/nsswitch.conf /etc/nsswitch.conf - -# KERBEROS -apt install -y krb5-user libpam-krb5 libsasl2-modules-gssapi-mit -cp .drone/krb5.conf /etc/krb5.conf - apt install -y netcat-openbsd - -sync_with auth1 - -rm -f /etc/krb5.keytab -cat </dev/null' HUP +trap 'running=0; kill -TERM $! 2>/dev/null' TERM INT +trap 'running=0; kill -KILL $! 2>/dev/null' EXIT + +while [ "$running" = 1 ]; do + "$@" & + wait + sleep "$TIMEOUT" +done diff --git a/.drone/uwldap_data.ldif b/.drone/uwldap_data.ldif new file mode 100644 index 0000000..89d2d66 --- /dev/null +++ b/.drone/uwldap_data.ldif @@ -0,0 +1,303 @@ +dn: ou=UWLDAP,dc=csclub,dc=internal +objectClass: organizationalUnit +ou: UWLDAP + +dn: uid=ctdalek,ou=UWLDAP,dc=csclub,dc=internal +displayName: Calum Dalek +givenName: Calum +sn: Dalek +cn: Calum Dalek +ou: MAT/Mathematics Computer Science +mailLocalAddress: ctdalek@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: ctdalek +mail: ctdalek@uwaterloo.internal + +dn: uid=regular1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular One +givenName: Regular +sn: One +cn: Regular One +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular1 +mail: regular1@uwaterloo.internal + +dn: uid=regular2,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular Two +givenName: Regular +sn: Two +cn: Regular Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular2 +mail: regular2@uwaterloo.internal + +dn: uid=exec1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec One +givenName: Exec +sn: One +cn: Exec One +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec1 +mail: exec1@uwaterloo.internal + +dn: uid=exec2,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec Two +givenName: Exec +sn: Two +cn: Exec Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec2 +mail: exec2@uwaterloo.internal + +dn: uid=ctdalek,ou=UWLDAP,dc=csclub,dc=internal +displayName: Calum Dalek +givenName: Calum +sn: Dalek +cn: Calum Dalek +ou: MAT/Mathematics Computer Science +mailLocalAddress: ctdalek@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: ctdalek +mail: ctdalek@uwaterloo.internal + +dn: uid=regular1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular One +givenName: Regular +sn: One +cn: Regular One +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular1 +mail: regular1@uwaterloo.internal + +dn: uid=regular2,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular Two +givenName: Regular +sn: Two +cn: Regular Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular2 +mail: regular2@uwaterloo.internal + +dn: uid=exec1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec One +givenName: Exec +sn: One +cn: Exec One +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec1 +mail: exec1@uwaterloo.internal + +dn: uid=ctdalek,ou=UWLDAP,dc=csclub,dc=internal +displayName: Calum Dalek +givenName: Calum +sn: Dalek +cn: Calum Dalek +ou: MAT/Mathematics Computer Science +mailLocalAddress: ctdalek@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: ctdalek +mail: ctdalek@uwaterloo.internal + +dn: uid=regular1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular One +givenName: Regular +sn: One +cn: Regular One +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular1 +mail: regular1@uwaterloo.internal + +dn: uid=regular2,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular Two +givenName: Regular +sn: Two +cn: Regular Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular2 +mail: regular2@uwaterloo.internal + +dn: uid=exec1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec One +givenName: Exec +sn: One +cn: Exec One +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec1 +mail: exec1@uwaterloo.internal + +dn: uid=ctdalek,ou=UWLDAP,dc=csclub,dc=internal +displayName: Calum Dalek +givenName: Calum +sn: Dalek +cn: Calum Dalek +ou: MAT/Mathematics Computer Science +mailLocalAddress: ctdalek@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: ctdalek +mail: ctdalek@uwaterloo.internal + +dn: uid=regular1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular One +givenName: Regular +sn: One +cn: Regular One +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular1 +mail: regular1@uwaterloo.internal + +dn: uid=regular2,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular Two +givenName: Regular +sn: Two +cn: Regular Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular2 +mail: regular2@uwaterloo.internal + +dn: uid=regular3,ou=UWLDAP,dc=csclub,dc=internal +displayName: Regular Three +givenName: Regular +sn: Three +cn: Regular Three +ou: MAT/Mathematics Computer Science +mailLocalAddress: regular3@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: regular3 +mail: regular3@uwaterloo.internal + +dn: uid=exec1,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec One +givenName: Exec +sn: One +cn: Exec One +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec1@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec1 +mail: exec1@uwaterloo.internal + +dn: uid=exec2,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec Two +givenName: Exec +sn: Two +cn: Exec Two +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec2@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec2 +mail: exec2@uwaterloo.internal + +dn: uid=exec3,ou=UWLDAP,dc=csclub,dc=internal +displayName: Exec Three +givenName: Exec +sn: Three +cn: Exec Three +ou: MAT/Mathematics Computer Science +mailLocalAddress: exec3@uwaterloo.internal +objectClass: inetLocalMailRecipient +objectClass: inetOrgPerson +objectClass: organizationalPerson +objectClass: person +objectClass: top +uid: exec3 +mail: exec3@uwaterloo.internal diff --git a/README.md b/README.md index 0ad696c..4779ae5 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,54 @@ club accounts and memberships. See [architecture.md](architecture.md) for an overview of its architecture. ## Development -First, make sure that you have installed the +### Docker +If you are not modifying code related to email or Mailman, then you may use +Docker containers instead, which are much easier to work with than the VM. + +First, make sure you create the virtualenv: +```sh +docker run --rm -v "$PWD:$PWD" -w "$PWD" -u $(id -u):$(id -g) python:3.7-buster \ + sh -c 'python -m venv venv && . venv/bin/activate && pip install -r requirements.txt -r dev-requirements.txt' +``` +Then bring up the containers: +```sh +docker-compose up -d # or without -d to run in the foreground +``` +This will create some containers with the bare minimum necessary for ceod to +run, and start ceod on each of phosphoric-acid, mail, and coffee container. +You can check the containers status using: +```sh +docker-compose logs -f +``` + +To use ceo, run the following: +```sh +docker-compose exec phosphoric-acid bash +su ctdalek +. venv/bin/activate +python -m ceo # the password is krb5 +``` +This should bring up the TUI. + +Normally, ceod should autoamtically restart when the source files are changed. +To manually restart the service, run: +```sh +docker-compose kill -s SIGHUP phosphoric-acid +``` + +To stop the containers, run: +```sh +docker-compose down +``` +Alternatively, if you started docker-compose in the foreground, just press Ctrl-C. + +### VM +If you need the full environment running in VM, follow the guide on [syscom dev environment](https://git.uwaterloo.ca/csc/syscom-dev-environment). This will setup all of the services needed for ceo to work. You should clone this repo in the phosphoric-acid container under ctdalek's home directory; you will then be able to access it from any container thanks to NFS. -### Environment setup Once you have the dev environment setup, there are a few more steps you'll need to do for ceo. @@ -129,7 +170,7 @@ pip install -r requirements.txt pip install -r dev-requirements.txt ``` -## Running the application +#### Running the application ceod is a distributed application, with instances on different hosts offering different services. Therefore, you will need to run ceod on multiple hosts. Currently, those are @@ -148,8 +189,16 @@ is running. Stop the flask app (Ctrl-C), run `clear_cache.sh`, then restart the app. ## Interacting with the application -The client part of ceo hasn't been written yet, so we'll use curl to -interact with ceod for now. +To use the TUI: +``` +python -m ceo +``` +To use the CLI: +``` +python -m ceo --help +``` + +Alternatively, you may use curl to send HTTP requests. ceod uses [SPNEGO](https://en.wikipedia.org/wiki/SPNEGO) for authentication, and TLS for confidentiality and integrity. In development mode, TLS can be diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..324d324 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,44 @@ +version: "3.6" + +x-common: &common + image: python:3.7-buster + volumes: + - .:$PWD + environment: + FLASK_APP: ceod.api + FLASK_ENV: development + working_dir: $PWD + entrypoint: + - ./docker-entrypoint.sh + +services: + auth1: + <<: *common + image: debian:buster + hostname: auth1 + command: auth1 + + coffee: + <<: *common + command: coffee + hostname: coffee + depends_on: + - auth1 + + mail: + <<: *common + command: mail + hostname: mail + depends_on: + - auth1 + + phosphoric-acid: + <<: *common + command: phosphoric-acid + hostname: phosphoric-acid + depends_on: + - auth1 + - coffee + - mail + +# vim: expandtab sw=2 ts=2 diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..9f19ecb --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/sh -e + +if ! [ -d venv ]; then + echo "You need to create the virtualenv first!" >&2 + exit 1 +fi + +host="$1" +[ -x ".drone/$host-setup.sh" ] && ".drone/$host-setup.sh" + +if [ "$host" = auth1 ]; then + exec sleep infinity +else + . venv/bin/activate + exec .drone/supervise.sh flask run -h 0.0.0.0 -p 9987 +fi diff --git a/tests/MockMailmanServer.py b/tests/MockMailmanServer.py index 561d1ff..d6ff78b 100644 --- a/tests/MockMailmanServer.py +++ b/tests/MockMailmanServer.py @@ -4,11 +4,12 @@ from aiohttp import web class MockMailmanServer: - def __init__(self): + def __init__(self, port=8001, prefix='/3.1'): + self.port = port self.app = web.Application() self.app.add_routes([ - web.post('/members', self.subscribe), - web.delete('/lists/{mailing_list}/member/{address}', self.unsubscribe), + web.post(prefix + '/members', self.subscribe), + web.delete(prefix + '/lists/{mailing_list}/member/{address}', self.unsubscribe), ]) self.runner = web.AppRunner(self.app) self.loop = asyncio.new_event_loop() @@ -24,7 +25,7 @@ class MockMailmanServer: def _start_loop(self): asyncio.set_event_loop(self.loop) self.loop.run_until_complete(self.runner.setup()) - site = web.TCPSite(self.runner, '127.0.0.1', 8002) + site = web.TCPSite(self.runner, '127.0.0.1', self.port) self.loop.run_until_complete(site.start()) self.loop.run_forever() @@ -67,3 +68,8 @@ class MockMailmanServer: }, status=404) subscribers.remove(subscriber) return web.json_response({'status': 'OK'}) + + +if __name__ == '__main__': + server = MockMailmanServer() + server.start() diff --git a/tests/MockSMTPServer.py b/tests/MockSMTPServer.py index 0a70c1f..e0cbbd3 100644 --- a/tests/MockSMTPServer.py +++ b/tests/MockSMTPServer.py @@ -1,3 +1,6 @@ +import os +import time + from aiosmtpd.controller import Controller @@ -25,3 +28,10 @@ class MockHandler: } self.mock_server.messages.append(msg) return '250 Message accepted for delivery' + + +if __name__ == '__main__': + assert os.geteuid() == 0 + server = MockSMTPServer('0.0.0.0', 25) + server.start() + time.sleep(1e6) diff --git a/tests/ceod_dev.ini b/tests/ceod_dev.ini index fea17d5..b4b6cc5 100644 --- a/tests/ceod_dev.ini +++ b/tests/ceod_dev.ini @@ -20,8 +20,8 @@ groups_base = ou=Group,dc=csclub,dc=internal sudo_base = ou=SUDOers,dc=csclub,dc=internal [uwldap] -server_url = ldap://uwldap.uwaterloo.ca -base = dc=uwaterloo,dc=ca +server_url = ldap://auth1.csclub.internal +base = ou=UWLDAP,dc=csclub,dc=internal [members] min_id = 20001 diff --git a/tests/ceod_test_local.ini b/tests/ceod_test_local.ini index 25d9dae..7927654 100644 --- a/tests/ceod_test_local.ini +++ b/tests/ceod_test_local.ini @@ -9,7 +9,7 @@ fs_root_host = phosphoric-acid mailman_host = phosphoric-acid database_host = phosphoric-acid use_https = false -port = 9987 +port = 9988 [ldap] admin_principal = ceod/admin @@ -40,7 +40,7 @@ smtp_url = smtp://localhost:8025 smtp_starttls = false [mailman3] -api_base_url = http://localhost:8002 +api_base_url = http://localhost:8001/3.1 api_username = restadmin api_password = mailman3 new_member_list = csc-general