db-api #10
|
@ -10,7 +10,7 @@ steps:
|
|||
# way to share system packages between steps
|
||||
commands:
|
||||
# install dependencies
|
||||
- apt update && apt install -y libkrb5-dev python3-dev
|
||||
- apt update && apt install -y libkrb5-dev libpq-dev python3-dev
|
||||
- python3 -m venv venv
|
||||
- . venv/bin/activate
|
||||
- pip install -r dev-requirements.txt
|
||||
|
@ -29,6 +29,11 @@ services:
|
|||
commands:
|
||||
- .drone/auth1-setup.sh
|
||||
- sleep infinity
|
||||
- name: coffee
|
||||
image: debian:buster
|
||||
commands:
|
||||
- .drone/coffee-setup.sh
|
||||
- sleep infinity
|
||||
|
||||
trigger:
|
||||
branch:
|
||||
|
|
|
@ -2,23 +2,7 @@
|
|||
|
||||
set -ex
|
||||
|
||||
# don't resolve container names to *real* CSC machines
|
||||
sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
|
||||
cat /tmp/resolv.conf > /etc/resolv.conf
|
||||
rm /tmp/resolv.conf
|
||||
|
||||
get_ip_addr() {
|
||||
getent hosts $1 | cut -d' ' -f1
|
||||
}
|
||||
|
||||
add_fqdn_to_hosts() {
|
||||
ip_addr=$1
|
||||
hostname=$2
|
||||
sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
|
||||
cat /tmp/hosts > /etc/hosts
|
||||
rm /tmp/hosts
|
||||
echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts
|
||||
}
|
||||
. .drone/common.sh
|
||||
|
||||
# set FQDN in /etc/hosts
|
||||
add_fqdn_to_hosts $(get_ip_addr $(hostname)) auth1
|
||||
|
|
|
@ -0,0 +1,48 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
. .drone/common.sh
|
||||
|
||||
# set FQDN in /etc/hosts
|
||||
add_fqdn_to_hosts $(get_ip_addr $(hostname)) coffee
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt update
|
||||
|
||||
apt install --no-install-recommends -y default-mysql-server postgresql
|
||||
|
||||
service mysql stop
|
||||
sed -E -i 's/^(bind-address[[:space:]]+= 127.0.0.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf
|
||||
service mysql start
|
||||
cat <<EOF | mysql
|
||||
CREATE USER 'mysql' IDENTIFIED BY 'mysql';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'mysql' WITH GRANT OPTION;
|
||||
EOF
|
||||
|
||||
service postgresql stop
|
||||
POSTGRES_DIR=/etc/postgresql/11/main
|
||||
cat <<EOF > $POSTGRES_DIR/pg_hba.conf
|
||||
# TYPE DATABASE USER ADDRESS METHOD
|
||||
local all postgres peer
|
||||
host all postgres 0.0.0.0/0 md5
|
||||
|
||||
local all all peer
|
||||
host all all localhost md5
|
||||
|
||||
local sameuser all md5
|
||||
host sameuser all 0.0.0.0/0 md5
|
||||
EOF
|
||||
grep -Eq "^listen_addresses = '*'$" $POSTGRES_DIR/postgresql.conf || \
|
||||
echo "listen_addresses = '*'" >> $POSTGRES_DIR/postgresql.conf
|
||||
service postgresql start
|
||||
su -c "
|
||||
cat <<EOF | psql
|
||||
ALTER USER postgres WITH PASSWORD 'postgres';
|
||||
REVOKE ALL ON SCHEMA public FROM public;
|
||||
GRANT ALL ON SCHEMA public TO postgres;
|
||||
EOF" postgres
|
||||
|
||||
# sync with phosphoric-acid
|
||||
apt install -y netcat-openbsd
|
||||
nc -l 0.0.0.0 9000
|
|
@ -0,0 +1,17 @@
|
|||
# don't resolve container names to *real* CSC machines
|
||||
sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
|
||||
cp /tmp/resolv.conf /etc/resolv.conf
|
||||
rm /tmp/resolv.conf
|
||||
|
||||
get_ip_addr() {
|
||||
getent hosts $1 | cut -d' ' -f1
|
||||
}
|
||||
|
||||
add_fqdn_to_hosts() {
|
||||
ip_addr=$1
|
||||
hostname=$2
|
||||
sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
|
||||
cp /tmp/hosts /etc/hosts
|
||||
rm /tmp/hosts
|
||||
echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts
|
||||
}
|
|
@ -2,27 +2,26 @@
|
|||
|
||||
set -ex
|
||||
|
||||
# don't resolve container names to *real* CSC machines
|
||||
sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
|
||||
cat /tmp/resolv.conf > /etc/resolv.conf
|
||||
rm /tmp/resolv.conf
|
||||
. .drone/common.sh
|
||||
|
||||
get_ip_addr() {
|
||||
getent hosts $1 | cut -d' ' -f1
|
||||
}
|
||||
|
||||
add_fqdn_to_hosts() {
|
||||
ip_addr=$1
|
||||
hostname=$2
|
||||
sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
|
||||
cat /tmp/hosts > /etc/hosts
|
||||
rm /tmp/hosts
|
||||
echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts
|
||||
sync_with() {
|
||||
host=$1
|
||||
synced=false
|
||||
# give it 5 minutes
|
||||
for i in {1..60}; do
|
||||
if nc -vz $host 9000 ; then
|
||||
synced=true
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
test $synced = true
|
||||
}
|
||||
|
||||
# set FQDN in /etc/hosts
|
||||
add_fqdn_to_hosts $(get_ip_addr $(hostname)) phosphoric-acid
|
||||
add_fqdn_to_hosts $(get_ip_addr auth1) auth1
|
||||
add_fqdn_to_hosts $(get_ip_addr coffee) coffee
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt update
|
||||
|
@ -41,18 +40,9 @@ cp .drone/nsswitch.conf /etc/nsswitch.conf
|
|||
apt install -y krb5-user libpam-krb5 libsasl2-modules-gssapi-mit
|
||||
cp .drone/krb5.conf /etc/krb5.conf
|
||||
|
||||
# sync with auth1
|
||||
apt install -y netcat-openbsd
|
||||
synced=false
|
||||
# give it 5 minutes
|
||||
for i in {1..60}; do
|
||||
if nc -vz auth1 9000 ; then
|
||||
synced=true
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
test $synced = true
|
||||
|
||||
sync_with auth1
|
||||
|
||||
rm -f /etc/krb5.keytab
|
||||
cat <<EOF | kadmin -p sysadmin/admin
|
||||
|
@ -66,6 +56,8 @@ ktadd ceod/admin
|
|||
EOF
|
||||
service nslcd start
|
||||
|
||||
sync_with coffee
|
||||
|
||||
# initialize the skel directory
|
||||
shopt -s dotglob
|
||||
mkdir -p /users/skel
|
||||
|
|
|
@ -4,3 +4,4 @@ __pycache__/
|
|||
.vscode/
|
||||
*.o
|
||||
*.so
|
||||
.idea/
|
||||
|
|
72
README.md
72
README.md
|
@ -33,7 +33,75 @@ On phosphoric-acid, you will additionally need to create a principal
|
|||
called `ceod/admin` (remember to addprinc **and** ktadd).
|
||||
|
||||
#### Database
|
||||
TODO - Andrew
|
||||
**Note**: The instructions below apply to the dev environment only; in
|
||||
production, the DB superusers should be restricted to the host where
|
||||
the DB is running.
|
||||
|
||||
Attach to the coffee container, run `mysql`, and run the following:
|
||||
|
||||
```
|
||||
CREATE USER 'mysql' IDENTIFIED BY 'mysql';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'mysql' WITH GRANT OPTION;
|
||||
```
|
||||
(In prod, the superuser should have '@localhost' appended to its name.)
|
||||
|
||||
Now open /etc/mysql/mariadb.conf.d/50-server.cnf and comment out the following line:
|
||||
```
|
||||
bind-address = 127.0.0.1
|
||||
```
|
||||
Then restart MariaDB:
|
||||
```
|
||||
systemctl restart mariadb
|
||||
```
|
||||
|
||||
Install PostgreSQL in the container:
|
||||
```
|
||||
apt install -y postgresql
|
||||
```
|
||||
Modify the superuser `postgres` for password authentication and restrict new users:
|
||||
```
|
||||
su postgres
|
||||
psql
|
||||
|
||||
ALTER USER postgres WITH PASSWORD 'postgres';
|
||||
REVOKE ALL ON SCHEMA public FROM public;
|
||||
GRANT ALL ON SCHEMA public TO postgres;
|
||||
```
|
||||
Create a new `pg_hba.conf`:
|
||||
```
|
||||
cd /etc/postgresql/<version>/<branch>/
|
||||
mv pg_hba.conf pg_hba.conf.old
|
||||
```
|
||||
```
|
||||
# new pg_hba.conf
|
||||
# TYPE DATABASE USER ADDRESS METHOD
|
||||
local all postgres peer
|
||||
host all postgres 0.0.0.0/0 md5
|
||||
|
||||
local all all peer
|
||||
host all all localhost md5
|
||||
|
||||
local sameuser all md5
|
||||
host sameuser all 0.0.0.0/0 md5
|
||||
```
|
||||
**Warning**: in prod, the postgres user should only be allowed to connect locally,
|
||||
so the relevant snippet in pg_hba.conf should look something like
|
||||
```
|
||||
local all postgres md5
|
||||
host all postgres localhost md5
|
||||
host all postgres 0.0.0.0/0 reject
|
||||
host all postgres ::/0 reject
|
||||
```
|
||||
Add the following to postgresql.conf:
|
||||
```
|
||||
listen_addresses = '*'
|
||||
```
|
||||
Now restart PostgreSQL:
|
||||
```
|
||||
systemctl restart postgresql
|
||||
```
|
||||
**In prod**, users can login remotely but superusers (`postgres` and `mysql`) are only
|
||||
allowed to login from the database host.
|
||||
|
||||
#### Mailman
|
||||
You should create the following mailing lists from the mail container:
|
||||
|
@ -54,7 +122,7 @@ messages get accepted (by default they get held).
|
|||
#### Dependencies
|
||||
Next, install and activate a virtualenv:
|
||||
```sh
|
||||
sudo apt install libkrb5-dev python3-dev
|
||||
sudo apt install libkrb5-dev libpq-dev python3-dev
|
||||
python3 -m venv venv
|
||||
. venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
|
|
@ -10,7 +10,7 @@ def http_request(method: str, path: str, **kwargs) -> requests.Response:
|
|||
client = component.getUtility(IHTTPClient)
|
||||
cfg = component.getUtility(IConfig)
|
||||
if path.startswith('/api/db'):
|
||||
host = cfg.get('ceod_db_host')
|
||||
host = cfg.get('ceod_database_host')
|
||||
delegate = False
|
||||
else:
|
||||
host = cfg.get('ceod_admin_host')
|
||||
|
|
|
@ -49,3 +49,18 @@ class UserNotSubscribedError(Exception):
|
|||
class NoSuchListError(Exception):
|
||||
def __init__(self):
|
||||
super().__init__('mailing list does not exist')
|
||||
|
||||
|
||||
class InvalidUsernameError(Exception):
|
||||
def __init__(self):
|
||||
super().__init__('Username contains characters that are not allowed')
|
||||
|
||||
|
||||
class DatabaseConnectionError(Exception):
|
||||
def __init__(self):
|
||||
super().__init__('unable to connect or authenticate to sql service')
|
||||
|
||||
|
||||
class DatabasePermissionError(Exception):
|
||||
def __init__(self):
|
||||
super().__init__('unable to perform action due to lack of permissions')
|
||||
|
|
|
@ -0,0 +1,18 @@
|
|||
from zope.interface import Attribute, Interface
|
||||
|
||||
|
||||
class IDatabaseService(Interface):
|
||||
"""Interface to create databases for users."""
|
||||
|
||||
type = Attribute('the type of databases that will be created')
|
||||
auth_username = Attribute('username to a privileged user on the database host')
|
||||
auth_password = Attribute('password to a privileged user on the database host')
|
||||
|
||||
def create_db(username: str) -> str:
|
||||
"""create a user and database and return the password"""
|
||||
|
||||
def reset_passwd(username: str) -> str:
|
||||
"""reset user password and return it"""
|
||||
|
||||
def delete_db(username: str):
|
||||
"""remove user and delete their database"""
|
|
@ -8,3 +8,4 @@ from .IUWLDAPService import IUWLDAPService
|
|||
from .IMailService import IMailService
|
||||
from .IMailmanService import IMailmanService
|
||||
from .IHTTPClient import IHTTPClient
|
||||
from .IDatabaseService import IDatabaseService
|
||||
|
|
|
@ -7,11 +7,12 @@ from zope import component
|
|||
|
||||
from .error_handlers import register_error_handlers
|
||||
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
|
||||
IMailmanService, IMailService, IUWLDAPService, IHTTPClient
|
||||
IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService
|
||||
from ceo_common.model import Config, HTTPClient, RemoteMailmanService
|
||||
from ceod.api.spnego import init_spnego
|
||||
from ceod.model import KerberosService, LDAPService, FileService, \
|
||||
MailmanService, MailService, UWLDAPService
|
||||
from ceod.db import MySQLService, PostgreSQLService
|
||||
|
||||
|
||||
def create_app(flask_config={}):
|
||||
|
@ -36,6 +37,10 @@ def create_app(flask_config={}):
|
|||
from ceod.api import mailman
|
||||
app.register_blueprint(mailman.bp, url_prefix='/api/mailman')
|
||||
|
||||
if hostname == cfg.get('ceod_database_host'):
|
||||
from ceod.api import database
|
||||
app.register_blueprint(database.bp, url_prefix='/api/db')
|
||||
|
||||
from ceod.api import groups
|
||||
app.register_blueprint(groups.bp, url_prefix='/api/groups')
|
||||
|
||||
|
@ -103,3 +108,13 @@ def register_services(app):
|
|||
# UWLDAPService
|
||||
uwldap_srv = UWLDAPService()
|
||||
component.provideUtility(uwldap_srv, IUWLDAPService)
|
||||
|
||||
# MySQLService
|
||||
if hostname == cfg.get('ceod_database_host'):
|
||||
mysql_srv = MySQLService()
|
||||
component.provideUtility(mysql_srv, IDatabaseService, 'mysql')
|
||||
|
||||
# PostgreSQLService
|
||||
if hostname == cfg.get('ceod_database_host'):
|
||||
psql_srv = PostgreSQLService()
|
||||
component.provideUtility(psql_srv, IDatabaseService, 'postgresql')
|
||||
|
|
|
@ -0,0 +1,104 @@
|
|||
from flask import Blueprint
|
||||
from zope import component
|
||||
from functools import wraps
|
||||
|
||||
from ceod.api.utils import authz_restrict_to_syscom, user_is_in_group, \
|
||||
requires_authentication_no_realm, development_only
|
||||
from ceo_common.errors import UserNotFoundError, DatabaseConnectionError, DatabasePermissionError, \
|
||||
InvalidUsernameError, UserAlreadyExistsError
|
||||
from ceo_common.interfaces import ILDAPService, IDatabaseService
|
||||
|
||||
|
||||
bp = Blueprint('db', __name__)
|
||||
|
||||
|
||||
def db_exception_handler(func):
|
||||
@wraps(func)
|
||||
def function(db_type: str, username: str):
|
||||
try:
|
||||
# Username should not contain symbols.
|
||||
# Underscores are allowed.
|
||||
for c in username:
|
||||
if not (c.isalnum() or c == '_'):
|
||||
raise InvalidUsernameError()
|
||||
ldap_srv = component.getUtility(ILDAPService)
|
||||
ldap_srv.get_user(username) # make sure user exists
|
||||
return func(db_type, username)
|
||||
except UserNotFoundError:
|
||||
return {'error': 'user not found'}, 404
|
||||
except UserAlreadyExistsError:
|
||||
return {'error': 'database user is already created'}, 409
|
||||
except InvalidUsernameError:
|
||||
return {'error': 'username contains invalid characters'}, 400
|
||||
except DatabaseConnectionError:
|
||||
return {'error': 'unable to connect or authenticate to sql server'}, 500
|
||||
except DatabasePermissionError:
|
||||
return {'error': 'unable to perform action due to permissions'}, 500
|
||||
return function
|
||||
|
||||
|
||||
@db_exception_handler
|
||||
def create_db_from_type(db_type: str, username: str):
|
||||
db_srv = component.getUtility(IDatabaseService, db_type)
|
||||
password = db_srv.create_db(username)
|
||||
return {'password': password}
|
||||
|
||||
|
||||
@db_exception_handler
|
||||
def reset_db_passwd_from_type(db_type: str, username: str):
|
||||
db_srv = component.getUtility(IDatabaseService, db_type)
|
||||
password = db_srv.reset_db_passwd(username)
|
||||
return {'password': password}
|
||||
|
||||
|
||||
@db_exception_handler
|
||||
def delete_db_from_type(db_type: str, username: str):
|
||||
db_srv = component.getUtility(IDatabaseService, db_type)
|
||||
db_srv.delete_db(username)
|
||||
return {'status': 'OK'}
|
||||
|
||||
|
||||
@bp.route('/mysql/<username>', methods=['POST'])
|
||||
@requires_authentication_no_realm
|
||||
def create_mysql_db(auth_user: str, username: str):
|
||||
if not (auth_user == username or user_is_in_group(auth_user, 'syscom')):
|
||||
return {'error': "not authorized to create databases for others"}, 403
|
||||
return create_db_from_type('mysql', username)
|
||||
|
||||
|
||||
@bp.route('/postgresql/<username>', methods=['POST'])
|
||||
@requires_authentication_no_realm
|
||||
def create_postgresql_db(auth_user: str, username: str):
|
||||
if not (auth_user == username or user_is_in_group(auth_user, 'syscom')):
|
||||
return {'error': "not authorized to create databases for others"}, 403
|
||||
return create_db_from_type('postgresql', username)
|
||||
|
||||
|
||||
@bp.route('/mysql/<username>/pwreset', methods=['POST'])
|
||||
@requires_authentication_no_realm
|
||||
def reset_mysql_db_passwd(auth_user: str, username: str):
|
||||
if not (auth_user == username or user_is_in_group(auth_user, 'syscom')):
|
||||
return {'error': "not authorized to request password reset for others"}, 403
|
||||
return reset_db_passwd_from_type('mysql', username)
|
||||
|
||||
|
||||
@bp.route('/postgresql/<username>/pwreset', methods=['POST'])
|
||||
@requires_authentication_no_realm
|
||||
def reset_postgresql_db_passwd(auth_user: str, username: str):
|
||||
if not (auth_user == username or user_is_in_group(auth_user, 'syscom')):
|
||||
return {'error': "not authorized to request password reset for others"}, 403
|
||||
return reset_db_passwd_from_type('postgresql', username)
|
||||
|
||||
|
||||
@bp.route('/mysql/<username>', methods=['DELETE'])
|
||||
@authz_restrict_to_syscom
|
||||
@development_only
|
||||
def delete_mysql_db(username: str):
|
||||
return delete_db_from_type('mysql', username)
|
||||
|
||||
|
||||
@bp.route('/postgresql/<username>', methods=['DELETE'])
|
||||
@authz_restrict_to_syscom
|
||||
@development_only
|
||||
def delete_postgresql_db(username: str):
|
||||
return delete_db_from_type('postgresql', username)
|
|
@ -0,0 +1,88 @@
|
|||
from zope.interface import implementer
|
||||
from zope import component
|
||||
from contextlib import contextmanager
|
||||
|
||||
from ceo_common.interfaces import IDatabaseService, IConfig
|
||||
from ceo_common.errors import DatabaseConnectionError, DatabasePermissionError, UserAlreadyExistsError, \
|
||||
UserNotFoundError
|
||||
from ceo_common.logger_factory import logger_factory
|
||||
from ceod.utils import gen_password
|
||||
from ceod.db.utils import response_is_empty
|
||||
|
||||
from mysql.connector import connect
|
||||
from mysql.connector.errors import InterfaceError, ProgrammingError
|
||||
|
||||
logger = logger_factory(__name__)
|
||||
|
||||
|
||||
@implementer(IDatabaseService)
|
||||
class MySQLService:
|
||||
|
||||
type = 'mysql'
|
||||
|
||||
def __init__(self):
|
||||
config = component.getUtility(IConfig)
|
||||
self.auth_username = config.get('mysql_username')
|
||||
self.auth_password = config.get('mysql_password')
|
||||
self.host = config.get('mysql_host')
|
||||
|
||||
@contextmanager
|
||||
def mysql_connection(self):
|
||||
try:
|
||||
with connect(
|
||||
host=self.host,
|
||||
user=self.auth_username,
|
||||
password=self.auth_password,
|
||||
) as con:
|
||||
yield con
|
||||
except InterfaceError as e:
|
||||
logger.error(e)
|
||||
raise DatabaseConnectionError()
|
||||
except ProgrammingError as e:
|
||||
logger.error(e)
|
||||
raise DatabasePermissionError()
|
||||
|
||||
def create_db(self, username: str) -> str:
|
||||
password = gen_password()
|
||||
search_for_user = f"SELECT user FROM mysql.user WHERE user='{username}'"
|
||||
search_for_db = f"SHOW DATABASES LIKE '{username}'"
|
||||
create_user = f"""
|
||||
CREATE USER '{username}'@'%' IDENTIFIED BY %(password)s;
|
||||
"""
|
||||
create_database = f"""
|
||||
CREATE DATABASE {username};
|
||||
GRANT ALL PRIVILEGES ON {username}.* TO '{username}'@'%';
|
||||
"""
|
||||
|
||||
with self.mysql_connection() as con, con.cursor() as cursor:
|
||||
if response_is_empty(search_for_user, con):
|
||||
cursor.execute(create_user, {'password': password})
|
||||
if response_is_empty(search_for_db, con):
|
||||
cursor.execute(create_database)
|
||||
else:
|
||||
raise UserAlreadyExistsError()
|
||||
return password
|
||||
|
||||
def reset_db_passwd(self, username: str) -> str:
|
||||
password = gen_password()
|
||||
search_for_user = f"SELECT user FROM mysql.user WHERE user='{username}'"
|
||||
reset_password = f"""
|
||||
ALTER USER '{username}'@'%' IDENTIFIED BY %(password)s
|
||||
"""
|
||||
|
||||
with self.mysql_connection() as con, con.cursor() as cursor:
|
||||
if not response_is_empty(search_for_user, con):
|
||||
cursor.execute(reset_password, {'password': password})
|
||||
else:
|
||||
raise UserNotFoundError(username)
|
||||
return password
|
||||
|
||||
def delete_db(self, username: str):
|
||||
drop_db = f"DROP DATABASE IF EXISTS {username}"
|
||||
drop_user = f"""
|
||||
DROP USER IF EXISTS '{username}'@'%';
|
||||
"""
|
||||
|
||||
with self.mysql_connection() as con, con.cursor() as cursor:
|
||||
cursor.execute(drop_db)
|
||||
cursor.execute(drop_user)
|
|
@ -0,0 +1,86 @@
|
|||
from zope.interface import implementer
|
||||
from zope import component
|
||||
from contextlib import contextmanager
|
||||
|
||||
from ceo_common.interfaces import IDatabaseService, IConfig
|
||||
from ceo_common.errors import DatabaseConnectionError, DatabasePermissionError, \
|
||||
UserAlreadyExistsError, UserNotFoundError
|
||||
from ceo_common.logger_factory import logger_factory
|
||||
from ceod.utils import gen_password
|
||||
from ceod.db.utils import response_is_empty
|
||||
|
||||
from psycopg2 import connect, OperationalError, ProgrammingError
|
||||
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
|
||||
|
||||
logger = logger_factory(__name__)
|
||||
|
||||
|
||||
@implementer(IDatabaseService)
|
||||
class PostgreSQLService:
|
||||
|
||||
type = 'postgresql'
|
||||
|
||||
def __init__(self):
|
||||
config = component.getUtility(IConfig)
|
||||
self.auth_username = config.get('postgresql_username')
|
||||
self.auth_password = config.get('postgresql_password')
|
||||
self.host = config.get('postgresql_host')
|
||||
|
||||
@contextmanager
|
||||
def psql_connection(self):
|
||||
con = None
|
||||
try:
|
||||
# Don't use the connection as a context manager, because that
|
||||
# creates a new transaction.
|
||||
con = connect(
|
||||
host=self.host,
|
||||
user=self.auth_username,
|
||||
password=self.auth_password,
|
||||
)
|
||||
con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
|
||||
yield con
|
||||
except OperationalError as e:
|
||||
logger.error(e)
|
||||
raise DatabaseConnectionError()
|
||||
except ProgrammingError as e:
|
||||
logger.error(e)
|
||||
raise DatabasePermissionError()
|
||||
finally:
|
||||
if con is not None:
|
||||
con.close()
|
||||
|
||||
def create_db(self, username: str) -> str:
|
||||
password = gen_password()
|
||||
search_for_user = f"SELECT FROM pg_roles WHERE rolname='{username}'"
|
||||
search_for_db = f"SELECT FROM pg_database WHERE datname='{username}'"
|
||||
create_user = f"CREATE USER {username} WITH PASSWORD %(password)s"
|
||||
create_database = f"CREATE DATABASE {username} OWNER {username}"
|
||||
revoke_perms = f"REVOKE ALL ON DATABASE {username} FROM PUBLIC"
|
||||
|
||||
with self.psql_connection() as con, con.cursor() as cursor:
|
||||
if not response_is_empty(search_for_user, con):
|
||||
raise UserAlreadyExistsError()
|
||||
cursor.execute(create_user, {'password': password})
|
||||
if response_is_empty(search_for_db, con):
|
||||
cursor.execute(create_database)
|
||||
cursor.execute(revoke_perms)
|
||||
return password
|
||||
|
||||
def reset_db_passwd(self, username: str) -> str:
|
||||
password = gen_password()
|
||||
search_for_user = f"SELECT FROM pg_roles WHERE rolname='{username}'"
|
||||
reset_password = f"ALTER USER {username} WITH PASSWORD %(password)s"
|
||||
|
||||
with self.psql_connection() as con, con.cursor() as cursor:
|
||||
if response_is_empty(search_for_user, con):
|
||||
raise UserNotFoundError(username)
|
||||
cursor.execute(reset_password, {'password': password})
|
||||
return password
|
||||
|
||||
def delete_db(self, username: str):
|
||||
drop_db = f"DROP DATABASE IF EXISTS {username}"
|
||||
drop_user = f"DROP USER IF EXISTS {username}"
|
||||
|
||||
with self.psql_connection() as con, con.cursor() as cursor:
|
||||
cursor.execute(drop_db)
|
||||
cursor.execute(drop_user)
|
|
@ -0,0 +1,2 @@
|
|||
from .MySQLService import MySQLService
|
||||
from .PostgreSQLService import PostgreSQLService
|
|
@ -0,0 +1,5 @@
|
|||
def response_is_empty(query: str, connection) -> bool:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(query)
|
||||
response = cursor.fetchall()
|
||||
return len(response) == 0
|
|
@ -7,3 +7,5 @@ requests==2.26.0
|
|||
requests-gssapi==1.2.3
|
||||
zope.component==5.0.1
|
||||
zope.interface==5.4.0
|
||||
mysql-connector-python==8.0.26
|
||||
psycopg2==2.9.1
|
|
@ -0,0 +1,120 @@
|
|||
import pytest
|
||||
|
||||
from ceod.model import User
|
||||
from mysql.connector import connect
|
||||
from mysql.connector.errors import ProgrammingError
|
||||
|
||||
|
||||
def test_api_create_mysql_db(cfg, client, g_admin_ctx, ldap_user, krb_user):
|
||||
uid = ldap_user.uid
|
||||
|
||||
with g_admin_ctx():
|
||||
user = User(uid='someone_else', cn='Some Name', terms=['s2021'])
|
||||
user.add_to_ldap()
|
||||
|
||||
# user should be able to create db for themselves
|
||||
status, data = client.post(f"/api/db/mysql/{uid}", json={}, principal=uid)
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
passwd = data['password']
|
||||
|
||||
# conflict if attempting to create db when already has one
|
||||
status, data = client.post(f"/api/db/mysql/{uid}", json={}, principal=uid)
|
||||
assert status == 409
|
||||
|
||||
# normal user cannot create db for others
|
||||
status, data = client.post("/api/db/mysql/someone_else", json={}, principal=uid)
|
||||
assert status == 403
|
||||
|
||||
# cannot create db for user not in ldap
|
||||
status, data = client.post("/api/db/mysql/user_not_found", json={})
|
||||
assert status == 404
|
||||
|
||||
# cannot create db when username contains symbols
|
||||
status, data = client.post("/api/db/mysql/!invalid", json={})
|
||||
assert status == 400
|
||||
|
||||
with connect(
|
||||
host=cfg.get('mysql_host'),
|
||||
user=uid,
|
||||
password=passwd,
|
||||
) as con, con.cursor() as cur:
|
||||
cur.execute("SHOW DATABASES")
|
||||
response = cur.fetchall()
|
||||
assert len(response) == 2
|
||||
|
||||
with pytest.raises(ProgrammingError):
|
||||
cur.execute("CREATE DATABASE new_db")
|
||||
|
||||
status, data = client.delete(f"/api/db/mysql/{uid}", json={})
|
||||
assert status == 200
|
||||
|
||||
# user should be deleted
|
||||
with pytest.raises(ProgrammingError):
|
||||
con = connect(
|
||||
host=cfg.get('mysql_host'),
|
||||
user=uid,
|
||||
password=passwd,
|
||||
)
|
||||
|
||||
# db should be deleted
|
||||
with connect(
|
||||
host=cfg.get('mysql_host'),
|
||||
user=cfg.get('mysql_username'),
|
||||
password=cfg.get('mysql_password'),
|
||||
) as con, con.cursor() as cur:
|
||||
cur.execute(f"SHOW DATABASES LIKE '{uid}'")
|
||||
response = cur.fetchall()
|
||||
assert len(response) == 0
|
||||
|
||||
with g_admin_ctx():
|
||||
user.remove_from_ldap()
|
||||
|
||||
|
||||
def test_api_passwd_reset_mysql(cfg, client, g_admin_ctx, ldap_user, krb_user):
|
||||
uid = ldap_user.uid
|
||||
|
||||
with g_admin_ctx():
|
||||
user = User(uid='someone_else', cn='Some Name', terms=['s2021'])
|
||||
user.add_to_ldap()
|
||||
|
||||
status, data = client.post(f"/api/db/mysql/{uid}", json={})
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
old_passwd = data['password']
|
||||
|
||||
con = connect(
|
||||
host=cfg.get('mysql_host'),
|
||||
user=uid,
|
||||
password=old_passwd,
|
||||
)
|
||||
con.close()
|
||||
|
||||
# normal user can get a password reset for themselves
|
||||
status, data = client.post(f"/api/db/mysql/{uid}/pwreset", json={}, principal=uid)
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
new_passwd = data['password']
|
||||
|
||||
assert old_passwd != new_passwd
|
||||
|
||||
# normal user cannot reset password for others
|
||||
status, data = client.post("/api/db/mysql/someone_else/pwreset", json={}, principal=uid)
|
||||
assert status == 403
|
||||
|
||||
# cannot password reset a user that does not have a database
|
||||
status, data = client.post("/api/db/mysql/someone_else/pwreset", json={})
|
||||
assert status == 404
|
||||
|
||||
con = connect(
|
||||
host=cfg.get('mysql_host'),
|
||||
user=uid,
|
||||
password=new_passwd,
|
||||
)
|
||||
con.close()
|
||||
|
||||
status, data = client.delete(f"/api/db/mysql/{uid}", json={})
|
||||
assert status == 200
|
||||
|
||||
with g_admin_ctx():
|
||||
user.remove_from_ldap()
|
|
@ -0,0 +1,123 @@
|
|||
import pytest
|
||||
|
||||
from ceod.model import User
|
||||
from psycopg2 import connect, OperationalError, ProgrammingError
|
||||
|
||||
|
||||
def test_api_create_psql_db(cfg, client, g_admin_ctx, ldap_user, krb_user):
|
||||
uid = ldap_user.uid
|
||||
|
||||
with g_admin_ctx():
|
||||
user = User(uid='someone_else', cn='Some Name', terms=['s2021'])
|
||||
user.add_to_ldap()
|
||||
|
||||
# user should be able to create db for themselves
|
||||
status, data = client.post(f"/api/db/postgresql/{uid}", json={}, principal=uid)
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
passwd = data['password']
|
||||
|
||||
# conflict if attempting to create db when already has one
|
||||
status, data = client.post(f"/api/db/postgresql/{uid}", json={}, principal=uid)
|
||||
assert status == 409
|
||||
|
||||
# normal user cannot create db for others
|
||||
status, data = client.post("/api/db/postgresql/someone_else", json={}, principal=uid)
|
||||
assert status == 403
|
||||
|
||||
# cannot create db for user not in ldap
|
||||
status, data = client.post("/api/db/postgresql/user_not_found", json={})
|
||||
assert status == 404
|
||||
|
||||
# cannot create db when username contains symbols
|
||||
status, data = client.post("/api/db/postgresql/!invalid", json={})
|
||||
assert status == 400
|
||||
|
||||
con = connect(
|
||||
host=cfg.get('postgresql_host'),
|
||||
user=uid,
|
||||
password=passwd,
|
||||
)
|
||||
con.autocommit = True
|
||||
with con.cursor() as cur:
|
||||
cur.execute("SELECT datname FROM pg_database")
|
||||
response = cur.fetchall()
|
||||
# 3 of the 4 are postgres, template0, template1
|
||||
assert len(response) == 4
|
||||
with pytest.raises(ProgrammingError):
|
||||
cur.execute("CREATE DATABASE new_db")
|
||||
con.close()
|
||||
|
||||
status, data = client.delete(f"/api/db/postgresql/{uid}", json={})
|
||||
assert status == 200
|
||||
|
||||
# user should be deleted
|
||||
with pytest.raises(OperationalError):
|
||||
con = connect(
|
||||
host=cfg.get('postgresql_host'),
|
||||
user=uid,
|
||||
password=passwd,
|
||||
)
|
||||
|
||||
# db should be deleted
|
||||
with connect(
|
||||
host=cfg.get('postgresql_host'),
|
||||
user=cfg.get('postgresql_username'),
|
||||
password=cfg.get('postgresql_password'),
|
||||
) as con, con.cursor() as cur:
|
||||
cur.execute(f"SELECT datname FROM pg_database WHERE datname = '{uid}'")
|
||||
response = cur.fetchall()
|
||||
assert len(response) == 0
|
||||
|
||||
with g_admin_ctx():
|
||||
user.remove_from_ldap()
|
||||
|
||||
|
||||
def test_api_passwd_reset_psql(cfg, client, g_admin_ctx, ldap_user, krb_user):
|
||||
uid = ldap_user.uid
|
||||
|
||||
with g_admin_ctx():
|
||||
user = User(uid='someone_else', cn='Some Name', terms=['s2021'])
|
||||
user.add_to_ldap()
|
||||
|
||||
status, data = client.post(f"/api/db/postgresql/{uid}", json={})
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
old_passwd = data['password']
|
||||
|
||||
con = connect(
|
||||
host=cfg.get('postgresql_host'),
|
||||
user=uid,
|
||||
password=old_passwd,
|
||||
)
|
||||
con.close()
|
||||
|
||||
# normal user can get a password reset for themselves
|
||||
status, data = client.post(f"/api/db/postgresql/{uid}/pwreset", json={}, principal=uid)
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
new_passwd = data['password']
|
||||
|
||||
assert old_passwd != new_passwd
|
||||
|
||||
# normal user cannot reset password for others
|
||||
status, data = client.post("/api/db/postgresql/someone_else/pwreset",
|
||||
json={}, principal=uid)
|
||||
assert status == 403
|
||||
|
||||
# cannot password reset a user that does not have a database
|
||||
status, data = client.post("/api/db/postgresql/someone_else/pwreset", json={})
|
||||
assert status == 404
|
||||
|
||||
con = connect(
|
||||
host=cfg.get('postgresql_host'),
|
||||
user=uid,
|
||||
password=new_passwd,
|
||||
)
|
||||
con.close()
|
||||
|
||||
status, data = client.delete(f"/api/db/postgresql/{uid}", json={})
|
||||
assert status == 200
|
||||
|
||||
with g_admin_ctx():
|
||||
user.remove_from_ldap()
|
|
@ -7,6 +7,7 @@ admin_host = phosphoric-acid
|
|||
# this is the host with NFS no_root_squash
|
||||
fs_root_host = phosphoric-acid
|
||||
mailman_host = mail
|
||||
database_host = coffee
|
||||
use_https = false
|
||||
port = 9987
|
||||
|
||||
|
@ -56,3 +57,13 @@ exec = exec
|
|||
required = president,vice-president,sysadmin
|
||||
available = president,vice-president,treasurer,secretary,
|
||||
sysadmin,cro,librarian,imapd,webmaster,offsck
|
||||
|
||||
[mysql]
|
||||
username = mysql
|
||||
password = mysql
|
||||
host = localhost
|
||||
|
||||
[postgresql]
|
||||
username = postgres
|
||||
password = postgres
|
||||
host = localhost
|
||||
|
|
|
@ -7,6 +7,7 @@ uw_domain = uwaterloo.internal
|
|||
admin_host = phosphoric-acid
|
||||
fs_root_host = phosphoric-acid
|
||||
mailman_host = phosphoric-acid
|
||||
database_host = phosphoric-acid
|
||||
use_https = false
|
||||
port = 9987
|
||||
|
||||
|
@ -55,3 +56,13 @@ exec = exec
|
|||
required = president,vice-president,sysadmin
|
||||
available = president,vice-president,treasurer,secretary,
|
||||
sysadmin,cro,librarian,imapd,webmaster,offsck
|
||||
|
||||
[mysql]
|
||||
username = mysql
|
||||
password = mysql
|
||||
host = coffee
|
||||
|
||||
[postgresql]
|
||||
username = postgres
|
||||
password = postgres
|
||||
host = coffee
|
||||
|
|
|
@ -6,6 +6,7 @@ import os
|
|||
import pwd
|
||||
import shutil
|
||||
import subprocess
|
||||
from subprocess import DEVNULL
|
||||
import sys
|
||||
import time
|
||||
from unittest.mock import patch, Mock
|
||||
|
@ -20,9 +21,11 @@ from zope import component
|
|||
|
||||
from .utils import gssapi_token_ctx, ccache_cleanup # noqa: F401
|
||||
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
|
||||
IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService
|
||||
IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
|
||||
IDatabaseService
|
||||
from ceo_common.model import Config, HTTPClient
|
||||
from ceod.api import create_app
|
||||
from ceod.db import MySQLService, PostgreSQLService
|
||||
from ceod.model import KerberosService, LDAPService, FileService, User, \
|
||||
MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService
|
||||
import ceod.utils as utils
|
||||
|
@ -241,6 +244,20 @@ def mail_srv(cfg, mock_mail_server):
|
|||
return _mail_srv
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def mysql_srv(cfg):
|
||||
mysql_srv = MySQLService()
|
||||
component.provideUtility(mysql_srv, IDatabaseService, 'mysql')
|
||||
return mysql_srv
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def postgresql_srv(cfg):
|
||||
psql_srv = PostgreSQLService()
|
||||
component.provideUtility(psql_srv, IDatabaseService, 'postgresql')
|
||||
return psql_srv
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope='session')
|
||||
def app(
|
||||
cfg,
|
||||
|
@ -250,6 +267,8 @@ def app(
|
|||
mailman_srv,
|
||||
uwldap_srv,
|
||||
mail_srv,
|
||||
mysql_srv,
|
||||
postgresql_srv,
|
||||
):
|
||||
app = create_app({'TESTING': True})
|
||||
return app
|
||||
|
@ -299,7 +318,11 @@ def ldap_user(simple_user, g_admin_ctx):
|
|||
|
||||
@pytest.fixture
|
||||
def krb_user(simple_user):
|
||||
simple_user.add_to_kerberos('krb5')
|
||||
# We don't want to use add_to_kerberos() here because that expires the
|
||||
# user's password, which we don't want for testing
|
||||
subprocess.run(
|
||||
['kadmin', '-k', '-p', 'ceod/admin', 'addprinc', '-pw', 'krb5',
|
||||
simple_user.uid], stdout=DEVNULL, check=True)
|
||||
yield simple_user
|
||||
simple_user.remove_from_kerberos()
|
||||
|
||||
|
|
Loading…
Reference in New Issue