fix tests

.drone.yml

@@ -29,6 +29,11 @@ services:
     commands:
       - .drone/auth1-setup.sh
       - sleep infinity
+  - name: coffee
+    image: debian:buster
+    commands:
+      - .drone/coffee-setup.sh
+      - sleep infinity
 
 trigger:
   branch:

.drone/auth1-setup.sh

@@ -2,23 +2,7 @@
 
 set -ex
 
-# don't resolve container names to *real* CSC machines
+. .drone/common.sh
-sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
-cat /tmp/resolv.conf > /etc/resolv.conf
-rm /tmp/resolv.conf
-
-get_ip_addr() {
-    getent hosts $1 | cut -d' ' -f1
-}
-
-add_fqdn_to_hosts() {
-    ip_addr=$1
-    hostname=$2
-    sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
-    cat /tmp/hosts > /etc/hosts
-    rm /tmp/hosts
-    echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts
-}
 
 # set FQDN in /etc/hosts
 add_fqdn_to_hosts $(get_ip_addr $(hostname)) auth1

.drone/coffee-setup.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -ex
+
+. .drone/common.sh
+
+# set FQDN in /etc/hosts
+add_fqdn_to_hosts $(get_ip_addr $(hostname)) coffee
+
+export DEBIAN_FRONTEND=noninteractive
+apt update
+
+apt install --no-install-recommends -y default-mysql-server postgresql
+
+service mysql stop
+sed -E -i 's/^(bind-address[[:space:]]+= 127.0.0.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf
+service mysql start
+cat <<EOF | mysql
+CREATE USER 'mysql' IDENTIFIED BY 'mysql';
+GRANT ALL PRIVILEGES ON *.* TO 'mysql' WITH GRANT OPTION;
+EOF
+
+service postgresql stop
+POSTGRES_DIR=/etc/postgresql/11/main
+cat <<EOF > $POSTGRES_DIR/pg_hba.conf
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+local   all             postgres                                peer
+host    all             postgres        0.0.0.0/0               md5
+
+local   all             all                                     peer
+host    all             all             localhost               md5
+
+local   sameuser        all                                     md5
+host    sameuser        all             0.0.0.0/0               md5
+EOF
+grep -Eq "^listen_addresses = '*'$" $POSTGRES_DIR/postgresql.conf || \
+     echo "listen_addresses = '*'" >> $POSTGRES_DIR/postgresql.conf
+service postgresql start
+su -c "
+cat <<EOF | psql
+ALTER USER postgres WITH PASSWORD 'postgres';
+REVOKE ALL ON SCHEMA public FROM public;
+GRANT ALL ON SCHEMA public TO postgres;
+EOF" postgres
+
+# sync with phosphoric-acid
+apt install -y netcat-openbsd
+nc -l 0.0.0.0 9000

.drone/common.sh

@@ -0,0 +1,17 @@
+# don't resolve container names to *real* CSC machines
+sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
+cp /tmp/resolv.conf /etc/resolv.conf
+rm /tmp/resolv.conf
+
+get_ip_addr() {
+    getent hosts $1 | cut -d' ' -f1
+}
+
+add_fqdn_to_hosts() {
+    ip_addr=$1
+    hostname=$2
+    sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
+    cp /tmp/hosts /etc/hosts
+    rm /tmp/hosts
+    echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts
+}

.drone/phosphoric-acid-setup.sh

@@ -2,27 +2,26 @@
 
 set -ex
 
-# don't resolve container names to *real* CSC machines
+. .drone/common.sh
-sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
-cat /tmp/resolv.conf > /etc/resolv.conf
-rm /tmp/resolv.conf
 
-get_ip_addr() {
+sync_with() {
-    getent hosts $1 | cut -d' ' -f1
+    host=$1
-}
+    synced=false
-
+    # give it 5 minutes
-add_fqdn_to_hosts() {
+    for i in {1..60}; do
-    ip_addr=$1
+        if nc -vz $host 9000 ; then
-    hostname=$2
+            synced=true
-    sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
+            break
-    cat /tmp/hosts > /etc/hosts
+        fi
-    rm /tmp/hosts
+        sleep 5
-    echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts
+    done
+    test $synced = true
 }
 
 # set FQDN in /etc/hosts
 add_fqdn_to_hosts $(get_ip_addr $(hostname)) phosphoric-acid
 add_fqdn_to_hosts $(get_ip_addr auth1) auth1
+add_fqdn_to_hosts $(get_ip_addr coffee) coffee
 
 export DEBIAN_FRONTEND=noninteractive
 apt update
@@ -41,18 +40,9 @@ cp .drone/nsswitch.conf /etc/nsswitch.conf
 apt install -y krb5-user libpam-krb5 libsasl2-modules-gssapi-mit
 cp .drone/krb5.conf /etc/krb5.conf
 
-# sync with auth1
 apt install -y netcat-openbsd
-synced=false
+
-# give it 5 minutes
+sync_with auth1
-for i in {1..60}; do
-    if nc -vz auth1 9000 ; then
-        synced=true
-        break
-    fi
-    sleep 5
-done
-test $synced = true
 
 rm -f /etc/krb5.keytab
 cat <<EOF | kadmin -p sysadmin/admin
@@ -66,6 +56,8 @@ ktadd ceod/admin
 EOF
 service nslcd start
 
+sync_with coffee
+
 # initialize the skel directory
 shopt -s dotglob
 mkdir -p /users/skel

ceod/api/database.py

@@ -2,8 +2,8 @@ from flask import Blueprint
 from zope import component
 from functools import wraps
 
-from ceod.api.utils import authz_restrict_to_syscom, user_is_in_group, requires_authentication_no_realm, \
+from ceod.api.utils import authz_restrict_to_syscom, user_is_in_group, \
-    development_only
+    requires_authentication_no_realm, development_only
 from ceo_common.errors import UserNotFoundError, DatabaseConnectionError, DatabasePermissionError, \
     InvalidUsernameError, UserAlreadyExistsError
 from ceo_common.interfaces import ILDAPService, IDatabaseService
@@ -16,8 +16,11 @@ def db_exception_handler(func):
     @wraps(func)
     def function(db_type: str, username: str):
         try:
-            if not username.isalnum():  # username should not contain symbols
+            # Username should not contain symbols.
-                raise InvalidUsernameError()
+            # Underscores are allowed.
+            for c in username:
+                if not (c.isalnum() or c == '_'):
+                    raise InvalidUsernameError()
             ldap_srv = component.getUtility(ILDAPService)
             ldap_srv.get_user(username)  # make sure user exists
             return func(db_type, username)
@@ -44,7 +47,7 @@ def create_db_from_type(db_type: str, username: str):
 @db_exception_handler
 def reset_db_passwd_from_type(db_type: str, username: str):
     db_srv = component.getUtility(IDatabaseService, db_type)
-    password = db_srv.reset_passwd(username)
+    password = db_srv.reset_db_passwd(username)
     return {'password': password}
 
 
@@ -52,6 +55,7 @@ def reset_db_passwd_from_type(db_type: str, username: str):
 def delete_db_from_type(db_type: str, username: str):
     db_srv = component.getUtility(IDatabaseService, db_type)
     db_srv.delete_db(username)
+    return {'status': 'OK'}
 
 
 @bp.route('/mysql/<username>', methods=['POST'])
@@ -90,11 +94,11 @@ def reset_postgresql_db_passwd(auth_user: str, username: str):
 @authz_restrict_to_syscom
 @development_only
 def delete_mysql_db(username: str):
-    delete_db_from_type('mysql', username)
+    return delete_db_from_type('mysql', username)
 
 
 @bp.route('/postgresql/<username>', methods=['DELETE'])
 @authz_restrict_to_syscom
 @development_only
 def delete_postgresql_db(username: str):
-    delete_db_from_type('postgresl', username)
+    return delete_db_from_type('postgresql', username)

ceod/db/MySQLService.py

@@ -5,12 +5,15 @@ from contextlib import contextmanager
 from ceo_common.interfaces import IDatabaseService, IConfig
 from ceo_common.errors import DatabaseConnectionError, DatabasePermissionError, UserAlreadyExistsError, \
     UserNotFoundError
+from ceo_common.logger_factory import logger_factory
 from ceod.utils import gen_password
 from ceod.db.utils import response_is_empty
 
 from mysql.connector import connect
 from mysql.connector.errors import InterfaceError, ProgrammingError
 
+logger = logger_factory(__name__)
+
 
 @implementer(IDatabaseService)
 class MySQLService:
@@ -21,39 +24,27 @@ class MySQLService:
         config = component.getUtility(IConfig)
         self.auth_username = config.get('mysql_username')
         self.auth_password = config.get('mysql_password')
-        try:
+        self.host = config.get('mysql_host')
-            test_user = "test_user_64559"
+
-            test_perms = f"""
+        # check that database is up and that we have admin rights
-            CREATE USER '{test_user}'@'localhost';
+        test_user = "test_user_64559"
-            CREATE DATABASE {test_user};
+        self.create_db(test_user)
-            GRANT ALL PRIVILEGES ON {test_user}.* TO '{test_user}'@'localhost';
+        self.delete_db(test_user)
-            DROP DATABASE {test_user};
-            DROP USER '{test_user}'@'localhost';
-            """
-            with connect(
-                    host='localhost',
-                    user=self.auth_username,
-                    password=self.auth_password,
-            ) as con:
-                with con.cursor() as cursor:
-                    cursor.execute(test_perms)
-        except InterfaceError:
-            raise Exception('unable to connect or authenticate to sql server')
-        except ProgrammingError:
-            raise Exception('insufficient permissions to create users and databases')
 
     @contextmanager
     def mysql_connection(self):
         try:
             with connect(
-                    host='localhost',
+                    host=self.host,
                     user=self.auth_username,
                     password=self.auth_password,
             ) as con:
                 yield con
-        except InterfaceError:
+        except InterfaceError as e:
+            logger.error(e)
             raise DatabaseConnectionError()
-        except ProgrammingError:
+        except ProgrammingError as e:
+            logger.error(e)
             raise DatabasePermissionError()
 
     def create_db(self, username: str) -> str:
@@ -61,49 +52,42 @@ class MySQLService:
         search_for_user = f"SELECT user FROM mysql.user WHERE user='{username}'"
         search_for_db = f"SHOW DATABASES LIKE '{username}'"
         create_user = f"""
-        CREATE USER '{username}'@'localhost' IDENTIFIED BY %(password)s;
         CREATE USER '{username}'@'%' IDENTIFIED BY %(password)s;
         """
         create_database = f"""
         CREATE DATABASE {username};
-        GRANT ALL PRIVILEGES ON {username}.* TO '{username}'@'localhost';
         GRANT ALL PRIVILEGES ON {username}.* TO '{username}'@'%';
         """
 
-        with self.mysql_connection() as con:
+        with self.mysql_connection() as con, con.cursor() as cursor:
-            with con.cursor() as cursor:
+            if response_is_empty(search_for_user, con):
-                if response_is_empty(search_for_user, con):
+                cursor.execute(create_user, {'password': password})
-                    cursor.execute(create_user, {'password': password})
+                if response_is_empty(search_for_db, con):
-                    if response_is_empty(search_for_db, con):
+                    cursor.execute(create_database)
-                        cursor.execute(create_database)
+            else:
-                else:
+                raise UserAlreadyExistsError()
-                    raise UserAlreadyExistsError()
+            return password
-                return password
 
     def reset_db_passwd(self, username: str) -> str:
         password = gen_password()
         search_for_user = f"SELECT user FROM mysql.user WHERE user='{username}'"
         reset_password = f"""
-        ALTER USER '{username}'@'localhost' IDENTIFIED BY %(password)s
         ALTER USER '{username}'@'%' IDENTIFIED BY %(password)s
         """
 
-        with self.mysql_connection() as con:
+        with self.mysql_connection() as con, con.cursor() as cursor:
-            with con.cursor() as cursor:
+            if not response_is_empty(search_for_user, con):
-                if not response_is_empty(search_for_user, con):
+                cursor.execute(reset_password, {'password': password})
-                    cursor.execute(reset_password, {'password': password})
+            else:
-                else:
+                raise UserNotFoundError(username)
-                    raise UserNotFoundError(username)
+            return password
-                return password
 
     def delete_db(self, username: str):
         drop_db = f"DROP DATABASE IF EXISTS {username}"
         drop_user = f"""
-        DROP USER IF EXISTS '{username}'@'localhost';
         DROP USER IF EXISTS '{username}'@'%';
         """
 
-        with self.mysql_connection() as con:
+        with self.mysql_connection() as con, con.cursor() as cursor:
-            with con.cursor() as cursor:
+            cursor.execute(drop_db)
-                cursor.execute(drop_db)
+            cursor.execute(drop_user)
-                cursor.execute(drop_user)

ceod/db/PostgreSQLService.py

@@ -3,14 +3,17 @@ from zope import component
 from contextlib import contextmanager
 
 from ceo_common.interfaces import IDatabaseService, IConfig
-from ceo_common.errors import DatabaseConnectionError, DatabasePermissionError, UserAlreadyExistsError, \
+from ceo_common.errors import DatabaseConnectionError, DatabasePermissionError, \
-    UserNotFoundError
+    UserAlreadyExistsError, UserNotFoundError
+from ceo_common.logger_factory import logger_factory
 from ceod.utils import gen_password
 from ceod.db.utils import response_is_empty
 
 from psycopg2 import connect, OperationalError, ProgrammingError
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 
+logger = logger_factory(__name__)
+
 
 @implementer(IDatabaseService)
 class PostgreSQLService:
@@ -21,80 +24,68 @@ class PostgreSQLService:
         config = component.getUtility(IConfig)
         self.auth_username = config.get('postgresql_username')
         self.auth_password = config.get('postgresql_password')
-        try:
+        self.host = config.get('postgresql_host')
-            test_user = "test_user_64559"
+
-            test_perms = f"""
+        # check that database is up and that we have admin rights
-            CREATE USER {test_user};
+        test_user = "test_user_64559"
-            CREATE DATABASE {test_user} OWNER {test_user};
+        self.create_db(test_user)
-            REVOKE ALL ON DATABASE {test_user} FROM PUBLIC;
+        self.delete_db(test_user)
-            DROP DATABASE {test_user};
-            DROP USER {test_user};
-            """
-            with connect(
-                    host='localhost',
-                    user=self.auth_username,
-                    password=self.auth_password,
-            ) as con:
-                with con.cursor() as cursor:
-                    cursor.execute(test_perms)
-        except OperationalError:
-            raise Exception('unable to connect or authenticate to sql server')
-        except ProgrammingError:
-            raise Exception('insufficient permissions to create users and databases')
 
     @contextmanager
     def psql_connection(self):
+        con = None
         try:
-            with connect(
+            # Don't use the connection as a context manager, because that
-                host='localhost',
+            # creates a new transaction.
+            con = connect(
+                host=self.host,
                 user=self.auth_username,
                 password=self.auth_password,
-            ) as con:
+            )
-                con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+            con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
-                yield con
+            yield con
-        except OperationalError:
+        except OperationalError as e:
+            logger.error(e)
             raise DatabaseConnectionError()
-        except ProgrammingError:
+        except ProgrammingError as e:
+            logger.error(e)
             raise DatabasePermissionError()
+        finally:
+            if con is not None:
+                con.close()
 
     def create_db(self, username: str) -> str:
         password = gen_password()
         search_for_user = f"SELECT FROM pg_roles WHERE rolname='{username}'"
         search_for_db = f"SELECT FROM pg_database WHERE datname='{username}'"
         create_user = f"CREATE USER {username} WITH PASSWORD %(password)s"
-        create_database = f"""
+        create_database = f"CREATE DATABASE {username} OWNER {username}"
-        CREATE DATABASE {username} OWNER {username};
+        revoke_perms = f"REVOKE ALL ON DATABASE {username} FROM PUBLIC"
-        REVOKE ALL ON DATABASE {username} FROM PUBLIC;
-        """
 
-        with self.psql_connection() as con:
+        with self.psql_connection() as con, con.cursor() as cursor:
-            with con.cursor() as cursor:
+            if not response_is_empty(search_for_user, con):
-                if response_is_empty(search_for_user, con):
+                raise UserAlreadyExistsError()
-                    cursor.execute(create_user, {'password': password})
+            cursor.execute(create_user, {'password': password})
-                    if response_is_empty(search_for_db, con):
+            if response_is_empty(search_for_db, con):
-                        cursor.execute(create_database)
+                cursor.execute(create_database)
-                else:
+                cursor.execute(revoke_perms)
-                    raise UserAlreadyExistsError()
+            return password
-                return password
 
     def reset_db_passwd(self, username: str) -> str:
         password = gen_password()
         search_for_user = f"SELECT FROM pg_roles WHERE rolname='{username}'"
         reset_password = f"ALTER USER {username} WITH PASSWORD %(password)s"
 
-        with self.psql_connection() as con:
+        with self.psql_connection() as con, con.cursor() as cursor:
-            with con.cursor() as cursor:
+            if response_is_empty(search_for_user, con):
-                if not response_is_empty(search_for_user, con):
+                raise UserNotFoundError(username)
-                    cursor.execute(reset_password, {'password': password})
+            cursor.execute(reset_password, {'password': password})
-                else:
+            return password
-                    raise UserNotFoundError(username)
-                return password
 
     def delete_db(self, username: str):
         drop_db = f"DROP DATABASE IF EXISTS {username}"
         drop_user = f"DROP USER IF EXISTS {username}"
 
-        with self.psql_connection() as con:
+        with self.psql_connection() as con, con.cursor() as cursor:
-            with con.cursor() as cursor:
+            cursor.execute(drop_db)
-                cursor.execute(drop_db)
+            cursor.execute(drop_user)
-                cursor.execute(drop_user)

tests/ceod/api/test_db_mysql.py

@@ -5,7 +5,7 @@ from mysql.connector import connect
 from mysql.connector.errors import InterfaceError, ProgrammingError
 
 
-def test_api_create_mysql_db(cfg, client, g_admin_ctx, ldap_user):
+def test_api_create_mysql_db(cfg, client, g_admin_ctx, ldap_user, krb_user):
     uid = ldap_user.uid
 
     with g_admin_ctx():
@@ -13,87 +13,85 @@ def test_api_create_mysql_db(cfg, client, g_admin_ctx, ldap_user):
         user.add_to_ldap()
 
     # user should be able to create db for themselves
-    status, data = client.post(f"/api/mysql/{uid}", json={}, principal=uid)
+    status, data = client.post(f"/api/db/mysql/{uid}", json={}, principal=uid)
     assert status == 200
     assert 'password' in data
     passwd = data['password']
 
     # conflict if attempting to create db when already has one
-    status, data = client.post(f"/api/mysql/{uid}", json={}, principal=uid)
+    status, data = client.post(f"/api/db/mysql/{uid}", json={}, principal=uid)
     assert status == 409
 
     # normal user cannot create db for others
-    status, data = client.post("/api/mysql/someone_else", json={}, principal=uid)
+    status, data = client.post("/api/db/mysql/someone_else", json={}, principal=uid)
     assert status == 403
 
     # cannot create db for user not in ldap
-    status, data = client.post("/api/mysql/user_not_found", json={})
+    status, data = client.post("/api/db/mysql/user_not_found", json={})
     assert status == 404
 
     # cannot create db when username contains symbols
-    status, data = client.post("/api/mysql/#invalid", json={})
+    status, data = client.post("/api/db/mysql/!invalid", json={})
     assert status == 400
 
     with connect(
-        host=cfg.get('ceod_database_host'),
+        host=cfg.get('mysql_host'),
         user=uid,
         password=passwd,
-    ) as con:
+    ) as con, con.cursor() as cur:
-        with con.cursor() as cur:
+        cur.execute("SHOW DATABASES")
-            cur.execute("SHOW DATABASES")
+        response = cur.fetchall()
-            response = cur.fetchall()
+        assert len(response) == 2
-            assert len(response) == 2
 
-            with pytest.raises(ProgrammingError):
+        with pytest.raises(ProgrammingError):
-                cur.execute("CREATE DATABASE new_db")
+            cur.execute("CREATE DATABASE new_db")
 
-    status, data = client.delete(f"/api/mysql/{uid}", json={})
+    status, data = client.delete(f"/api/db/mysql/{uid}", json={})
     assert status == 200
 
     # user should be deleted
-    with pytest.raises(InterfaceError):
+    with pytest.raises(ProgrammingError):
         con = connect(
-            host=cfg.get('ceod_database_host'),
+            host=cfg.get('mysql_host'),
             user=uid,
             password=passwd,
         )
 
     # db should be deleted
     with connect(
-            host=cfg.get('ceod_database_host'),
+            host=cfg.get('mysql_host'),
             user=cfg.get('mysql_username'),
             password=cfg.get('mysql_password'),
-    ) as con:
+    ) as con, con.cursor() as cur:
-        with con.cursor() as cur:
+        cur.execute(f"SHOW DATABASES LIKE '{uid}'")
-            cur.execute(f"SHOW DATABASES LIKE '{uid}'")
+        response = cur.fetchall()
-            response = cur.fetchall()
+        assert len(response) == 0
-            assert len(response) == 0
 
     with g_admin_ctx():
         user.remove_from_ldap()
 
 
-def test_api_passwd_reset_mysql(cfg, client, g_admin_ctx, ldap_user):
+def test_api_passwd_reset_mysql(cfg, client, g_admin_ctx, ldap_user, krb_user):
     uid = ldap_user.uid
 
     with g_admin_ctx():
         user = User(uid='someone_else', cn='Some Name', terms=['s2021'])
         user.add_to_ldap()
 
-    status, data = client.post(f"/api/mysql/{uid}", json={})
+    status, data = client.post(f"/api/db/mysql/{uid}", json={})
     assert status == 200
     assert 'password' in data
     old_passwd = data['password']
 
     con = connect(
-        host=cfg.get('ceod_database_host'),
+        host=cfg.get('mysql_host'),
         user=uid,
         password=old_passwd,
     )
     con.close()
 
     # normal user can get a password reset for themselves
-    status, data = client.post(f"/api/mysql/{uid}/pwreset", json={}, principal=uid)
+    status, data = client.post(f"/api/db/mysql/{uid}/pwreset", json={}, principal=uid)
     assert status == 200
     assert 'password' in data
     new_passwd = data['password']
@@ -101,21 +99,21 @@ def test_api_passwd_reset_mysql(cfg, client, g_admin_ctx, ldap_user):
     assert old_passwd != new_passwd
 
     # normal user cannot reset password for others
-    status, data = client.post(f"/api/mysql/{uid}/pwreset", json={}, principal='someone_else')
+    status, data = client.post(f"/api/db/mysql/someone_else/pwreset", json={}, principal=uid)
     assert status == 403
 
     # cannot password reset a user that does not have a database
-    status, data = client.post("/api/mysql/someone_else/pwreset", json={})
+    status, data = client.post("/api/db/mysql/someone_else/pwreset", json={})
     assert status == 404
 
     con = connect(
-        host=cfg.get('ceod_database_host'),
+        host=cfg.get('mysql_host'),
         user=uid,
         password=new_passwd,
     )
     con.close()
 
-    status, data = client.delete(f"/api/mysql/{uid}", json={})
+    status, data = client.delete(f"/api/db/mysql/{uid}", json={})
     assert status == 200
 
     with g_admin_ctx():

tests/ceod/api/test_db_psql.py

@@ -4,7 +4,7 @@ from ceod.model import User
 from psycopg2 import connect, OperationalError, ProgrammingError
 
 
-def test_api_create_psql_db(cfg, client, g_admin_ctx, ldap_user):
+def test_api_create_psql_db(cfg, client, g_admin_ctx, ldap_user, krb_user):
     uid = ldap_user.uid
 
     with g_admin_ctx():
@@ -12,87 +12,88 @@ def test_api_create_psql_db(cfg, client, g_admin_ctx, ldap_user):
         user.add_to_ldap()
 
     # user should be able to create db for themselves
-    status, data = client.post(f"/api/postgresql/{uid}", json={}, principal=uid)
+    status, data = client.post(f"/api/db/postgresql/{uid}", json={}, principal=uid)
     assert status == 200
     assert 'password' in data
     passwd = data['password']
 
     # conflict if attempting to create db when already has one
-    status, data = client.post(f"/api/postgresql/{uid}", json={}, principal=uid)
+    status, data = client.post(f"/api/db/postgresql/{uid}", json={}, principal=uid)
     assert status == 409
 
     # normal user cannot create db for others
-    status, data = client.post("/api/postgresql/someone_else", json={}, principal=uid)
+    status, data = client.post("/api/db/postgresql/someone_else", json={}, principal=uid)
     assert status == 403
 
     # cannot create db for user not in ldap
-    status, data = client.post("/api/postgresql/user_not_found", json={})
+    status, data = client.post("/api/db/postgresql/user_not_found", json={})
     assert status == 404
 
     # cannot create db when username contains symbols
-    status, data = client.post("/api/postgresql/#invalid", json={})
+    status, data = client.post("/api/db/postgresql/!invalid", json={})
     assert status == 400
 
-    with connect(
+    con = connect(
-        host=cfg.get('ceod_database_host'),
+        host=cfg.get('postgresql_host'),
         user=uid,
         password=passwd,
-    ) as con:
+    )
-        with con.cursor() as cur:
+    con.autocommit = True
-            cur.execute("SHOW DATABASES")
+    with con.cursor() as cur:
-            response = cur.fetchall()
+        cur.execute("SELECT datname FROM pg_database")
-            assert len(response) == 2
+        response = cur.fetchall()
+        # 3 of the 4 are postgres, template0, template1
+        assert len(response) == 4
+        with pytest.raises(ProgrammingError):
+            cur.execute("CREATE DATABASE new_db")
+    con.close()
 
-            with pytest.raises(ProgrammingError):
+    status, data = client.delete(f"/api/db/postgresql/{uid}", json={})
-                cur.execute("CREATE DATABASE new_db")
-
-    status, data = client.delete(f"/api/postgresql/{uid}", json={})
     assert status == 200
 
     # user should be deleted
     with pytest.raises(OperationalError):
         con = connect(
-            host=cfg.get('ceod_database_host'),
+            host=cfg.get('postgresql_host'),
             user=uid,
             password=passwd,
         )
 
     # db should be deleted
     with connect(
-            host=cfg.get('ceod_database_host'),
+            host=cfg.get('postgresql_host'),
             user=cfg.get('postgresql_username'),
             password=cfg.get('postgresql_password'),
-    ) as con:
+    ) as con, con.cursor() as cur:
-        with con.cursor() as cur:
+        cur.execute(f"SELECT datname FROM pg_database WHERE datname = '{uid}'")
-            cur.execute(f"SHOW DATABASES LIKE '{uid}'")
+        response = cur.fetchall()
-            response = cur.fetchall()
+        assert len(response) == 0
-            assert len(response) == 0
 
     with g_admin_ctx():
         user.remove_from_ldap()
 
 
-def test_api_passwd_reset_psql(cfg, client, g_admin_ctx, ldap_user):
+def test_api_passwd_reset_psql(cfg, client, g_admin_ctx, ldap_user, krb_user):
     uid = ldap_user.uid
 
     with g_admin_ctx():
         user = User(uid='someone_else', cn='Some Name', terms=['s2021'])
         user.add_to_ldap()
 
-    status, data = client.post(f"/api/postgresql/{uid}", json={})
+    status, data = client.post(f"/api/db/postgresql/{uid}", json={})
     assert status == 200
     assert 'password' in data
     old_passwd = data['password']
 
     con = connect(
-        host=cfg.get('ceod_database_host'),
+        host=cfg.get('postgresql_host'),
         user=uid,
         password=old_passwd,
     )
     con.close()
 
     # normal user can get a password reset for themselves
-    status, data = client.post(f"/api/postgresql/{uid}/pwreset", json={}, principal=uid)
+    status, data = client.post(f"/api/db/postgresql/{uid}/pwreset", json={}, principal=uid)
     assert status == 200
     assert 'password' in data
     new_passwd = data['password']
@@ -100,21 +101,22 @@ def test_api_passwd_reset_psql(cfg, client, g_admin_ctx, ldap_user):
     assert old_passwd != new_passwd
 
     # normal user cannot reset password for others
-    status, data = client.post(f"/api/postgresql/{uid}/pwreset", json={}, principal='someone_else')
+    status, data = client.post("/api/db/postgresql/someone_else/pwreset",
+                               json={}, principal=uid)
     assert status == 403
 
     # cannot password reset a user that does not have a database
-    status, data = client.post("/api/postgresql/someone_else/pwreset", json={})
+    status, data = client.post("/api/db/postgresql/someone_else/pwreset", json={})
     assert status == 404
 
     con = connect(
-        host=cfg.get('ceod_database_host'),
+        host=cfg.get('postgresql_host'),
         user=uid,
         password=new_passwd,
     )
     con.close()
 
-    status, data = client.delete(f"/api/postgresql/{uid}", json={})
+    status, data = client.delete(f"/api/db/postgresql/{uid}", json={})
     assert status == 200
 
     with g_admin_ctx():

tests/ceod_dev.ini

@@ -61,8 +61,9 @@ available = president,vice-president,treasurer,secretary,
 [mysql]
 username = mysql
 password = mysql
+host = localhost
 
 [postgresql]
 username = postgres
 password = postgres
-
+host = localhost

tests/ceod_test_local.ini

@@ -60,8 +60,9 @@ available = president,vice-president,treasurer,secretary,
 [mysql]
 username = mysql
 password = mysql
+host = coffee
 
 [postgresql]
 username = postgres
 password = postgres
-
+host = coffee

tests/conftest.py

@@ -6,6 +6,7 @@ import os
 import pwd
 import shutil
 import subprocess
+from subprocess import DEVNULL
 import sys
 import time
 from unittest.mock import patch, Mock
@@ -19,9 +20,11 @@ from zope import component
 
 from .utils import gssapi_creds_ctx, ccache_cleanup  # noqa: F401
 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
-    IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService
+    IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
+    IDatabaseService
 from ceo_common.model import Config, HTTPClient
 from ceod.api import create_app
+from ceod.db import MySQLService, PostgreSQLService
 from ceod.model import KerberosService, LDAPService, FileService, User, \
     MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService
 import ceod.utils as utils
@@ -239,6 +242,20 @@ def mail_srv(cfg, mock_mail_server):
     return _mail_srv
 
 
+@pytest.fixture(scope='session')
+def mysql_srv(cfg):
+    mysql_srv = MySQLService()
+    component.provideUtility(mysql_srv, IDatabaseService, 'mysql')
+    return mysql_srv
+
+
+@pytest.fixture(scope='session')
+def postgresql_srv(cfg):
+    psql_srv = PostgreSQLService()
+    component.provideUtility(psql_srv, IDatabaseService, 'postgresql')
+    return psql_srv
+
+
 @pytest.fixture(autouse=True, scope='session')
 def app(
     cfg,
@@ -248,6 +265,8 @@ def app(
     mailman_srv,
     uwldap_srv,
     mail_srv,
+    mysql_srv,
+    postgresql_srv,
 ):
     app = create_app({'TESTING': True})
     return app
@@ -297,7 +316,11 @@ def ldap_user(simple_user, g_admin_ctx):
 
 @pytest.fixture
 def krb_user(simple_user):
-    simple_user.add_to_kerberos('krb5')
+    # We don't want to use add_to_kerberos() here because that expires the
+    # user's password, which we don't want for testing
+    subprocess.run(
+        ['kadmin', '-k', '-p', 'ceod/admin', 'addprinc', '-pw', 'krb5',
+         simple_user.uid], stdout=DEVNULL, check=True)
     yield simple_user
     simple_user.remove_from_kerberos()