db-api #10
|
@ -10,7 +10,7 @@ steps:
|
|||
# way to share system packages between steps
|
||||
commands:
|
||||
# install dependencies
|
||||
- apt update && apt install -y libkrb5-dev python3-dev
|
||||
- apt update && apt install -y libkrb5-dev libpq-dev python3-dev
|
||||
- python3 -m venv venv
|
||||
- . venv/bin/activate
|
||||
- pip install -r dev-requirements.txt
|
||||
|
@ -29,6 +29,11 @@ services:
|
|||
commands:
|
||||
- .drone/auth1-setup.sh
|
||||
- sleep infinity
|
||||
- name: coffee
|
||||
image: debian:buster
|
||||
commands:
|
||||
- .drone/coffee-setup.sh
|
||||
- sleep infinity
|
||||
|
||||
trigger:
|
||||
branch:
|
||||
|
|
|
@ -2,23 +2,7 @@
|
|||
|
||||
set -ex
|
||||
|
||||
# don't resolve container names to *real* CSC machines
|
||||
sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
|
||||
cat /tmp/resolv.conf > /etc/resolv.conf
|
||||
rm /tmp/resolv.conf
|
||||
|
||||
get_ip_addr() {
|
||||
getent hosts $1 | cut -d' ' -f1
|
||||
}
|
||||
|
||||
add_fqdn_to_hosts() {
|
||||
ip_addr=$1
|
||||
hostname=$2
|
||||
sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
|
||||
cat /tmp/hosts > /etc/hosts
|
||||
rm /tmp/hosts
|
||||
echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts
|
||||
}
|
||||
. .drone/common.sh
|
||||
|
||||
# set FQDN in /etc/hosts
|
||||
add_fqdn_to_hosts $(get_ip_addr $(hostname)) auth1
|
||||
|
|
|
@ -0,0 +1,48 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
. .drone/common.sh
|
||||
|
||||
# set FQDN in /etc/hosts
|
||||
add_fqdn_to_hosts $(get_ip_addr $(hostname)) coffee
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt update
|
||||
|
||||
apt install --no-install-recommends -y default-mysql-server postgresql
|
||||
|
||||
service mysql stop
|
||||
sed -E -i 's/^(bind-address[[:space:]]+= 127.0.0.1)$/#\1/' /etc/mysql/mariadb.conf.d/50-server.cnf
|
||||
service mysql start
|
||||
cat <<EOF | mysql
|
||||
CREATE USER 'mysql' IDENTIFIED BY 'mysql';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'mysql' WITH GRANT OPTION;
|
||||
EOF
|
||||
|
||||
service postgresql stop
|
||||
POSTGRES_DIR=/etc/postgresql/11/main
|
||||
cat <<EOF > $POSTGRES_DIR/pg_hba.conf
|
||||
# TYPE DATABASE USER ADDRESS METHOD
|
||||
local all postgres peer
|
||||
host all postgres 0.0.0.0/0 md5
|
||||
|
||||
local all all peer
|
||||
host all all localhost md5
|
||||
|
||||
local sameuser all md5
|
||||
host sameuser all 0.0.0.0/0 md5
|
||||
EOF
|
||||
grep -Eq "^listen_addresses = '*'$" $POSTGRES_DIR/postgresql.conf || \
|
||||
echo "listen_addresses = '*'" >> $POSTGRES_DIR/postgresql.conf
|
||||
service postgresql start
|
||||
su -c "
|
||||
cat <<EOF | psql
|
||||
ALTER USER postgres WITH PASSWORD 'postgres';
|
||||
REVOKE ALL ON SCHEMA public FROM public;
|
||||
GRANT ALL ON SCHEMA public TO postgres;
|
||||
EOF" postgres
|
||||
|
||||
# sync with phosphoric-acid
|
||||
apt install -y netcat-openbsd
|
||||
nc -l 0.0.0.0 9000
|
|
@ -0,0 +1,17 @@
|
|||
# don't resolve container names to *real* CSC machines
|
||||
sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
|
||||
cp /tmp/resolv.conf /etc/resolv.conf
|
||||
rm /tmp/resolv.conf
|
||||
|
||||
get_ip_addr() {
|
||||
getent hosts $1 | cut -d' ' -f1
|
||||
}
|
||||
|
||||
add_fqdn_to_hosts() {
|
||||
ip_addr=$1
|
||||
hostname=$2
|
||||
sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
|
||||
cp /tmp/hosts /etc/hosts
|
||||
rm /tmp/hosts
|
||||
echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts
|
||||
}
|
|
@ -2,27 +2,26 @@
|
|||
|
||||
set -ex
|
||||
|
||||
# don't resolve container names to *real* CSC machines
|
||||
sed -E '/^(domain|search)[[:space:]]+csclub.uwaterloo.ca/d' /etc/resolv.conf > /tmp/resolv.conf
|
||||
cat /tmp/resolv.conf > /etc/resolv.conf
|
||||
rm /tmp/resolv.conf
|
||||
. .drone/common.sh
|
||||
|
||||
get_ip_addr() {
|
||||
getent hosts $1 | cut -d' ' -f1
|
||||
}
|
||||
|
||||
add_fqdn_to_hosts() {
|
||||
ip_addr=$1
|
||||
hostname=$2
|
||||
sed -E "/${ip_addr}.*\\b${hostname}\\b/d" /etc/hosts > /tmp/hosts
|
||||
cat /tmp/hosts > /etc/hosts
|
||||
rm /tmp/hosts
|
||||
echo "$ip_addr $hostname.csclub.internal $hostname" >> /etc/hosts
|
||||
sync_with() {
|
||||
host=$1
|
||||
synced=false
|
||||
# give it 5 minutes
|
||||
for i in {1..60}; do
|
||||
if nc -vz $host 9000 ; then
|
||||
synced=true
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
test $synced = true
|
||||
}
|
||||
|
||||
# set FQDN in /etc/hosts
|
||||
add_fqdn_to_hosts $(get_ip_addr $(hostname)) phosphoric-acid
|
||||
add_fqdn_to_hosts $(get_ip_addr auth1) auth1
|
||||
add_fqdn_to_hosts $(get_ip_addr coffee) coffee
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt update
|
||||
|
@ -41,18 +40,9 @@ cp .drone/nsswitch.conf /etc/nsswitch.conf
|
|||
apt install -y krb5-user libpam-krb5 libsasl2-modules-gssapi-mit
|
||||
cp .drone/krb5.conf /etc/krb5.conf
|
||||
|
||||
# sync with auth1
|
||||
apt install -y netcat-openbsd
|
||||
synced=false
|
||||
# give it 5 minutes
|
||||
for i in {1..60}; do
|
||||
if nc -vz auth1 9000 ; then
|
||||
synced=true
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
test $synced = true
|
||||
|
||||
sync_with auth1
|
||||
|
||||
rm -f /etc/krb5.keytab
|
||||
cat <<EOF | kadmin -p sysadmin/admin
|
||||
|
@ -66,6 +56,8 @@ ktadd ceod/admin
|
|||
EOF
|
||||
service nslcd start
|
||||
|
||||
sync_with coffee
|
||||
|
||||
# initialize the skel directory
|
||||
shopt -s dotglob
|
||||
mkdir -p /users/skel
|
||||
|
|
|
@ -4,3 +4,4 @@ __pycache__/
|
|||
.vscode/
|
||||
*.o
|
||||
*.so
|
||||
.idea/
|
||||
|
|
72
README.md
|
@ -33,7 +33,75 @@ On phosphoric-acid, you will additionally need to create a principal
|
|||
called `ceod/admin` (remember to addprinc **and** ktadd).
|
||||
|
||||
#### Database
|
||||
TODO - Andrew
|
||||
**Note**: The instructions below apply to the dev environment only; in
|
||||
production, the DB superusers should be restricted to the host where
|
||||
the DB is running.
|
||||
|
||||
Attach to the coffee container, run `mysql`, and run the following:
|
||||
|
||||
```
|
||||
CREATE USER 'mysql' IDENTIFIED BY 'mysql';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'mysql' WITH GRANT OPTION;
|
||||
```
|
||||
(In prod, the superuser should have '@localhost' appended to its name.)
|
||||
|
||||
Now open /etc/mysql/mariadb.conf.d/50-server.cnf and comment out the following line:
|
||||
```
|
||||
bind-address = 127.0.0.1
|
||||
```
|
||||
Then restart MariaDB:
|
||||
a268wang marked this conversation as resolved
Outdated
|
||||
```
|
||||
systemctl restart mariadb
|
||||
```
|
||||
|
||||
Install PostgreSQL in the container:
|
||||
```
|
||||
apt install -y postgresql
|
||||
```
|
||||
Modify the superuser `postgres` for password authentication and restrict new users:
|
||||
```
|
||||
su postgres
|
||||
psql
|
||||
|
||||
ALTER USER postgres WITH PASSWORD 'postgres';
|
||||
REVOKE ALL ON SCHEMA public FROM public;
|
||||
GRANT ALL ON SCHEMA public TO postgres;
|
||||
```
|
||||
Create a new `pg_hba.conf`:
|
||||
```
|
||||
cd /etc/postgresql/<version>/<branch>/
|
||||
mv pg_hba.conf pg_hba.conf.old
|
||||
```
|
||||
```
|
||||
# new pg_hba.conf
|
||||
# TYPE DATABASE USER ADDRESS METHOD
|
||||
local all postgres peer
|
||||
host all postgres 0.0.0.0/0 md5
|
||||
|
||||
local all all peer
|
||||
host all all localhost md5
|
||||
|
||||
local sameuser all md5
|
||||
host sameuser all 0.0.0.0/0 md5
|
||||
```
|
||||
**Warning**: in prod, the postgres user should only be allowed to connect locally,
|
||||
so the relevant snippet in pg_hba.conf should look something like
|
||||
```
|
||||
local all postgres md5
|
||||
host all postgres localhost md5
|
||||
host all postgres 0.0.0.0/0 reject
|
||||
host all postgres ::/0 reject
|
||||
```
|
||||
Add the following to postgresql.conf:
|
||||
```
|
||||
listen_addresses = '*'
|
||||
```
|
||||
Now restart PostgreSQL:
|
||||
```
|
||||
systemctl restart postgresql
|
||||
```
|
||||
**In prod**, users can login remotely but superusers (`postgres` and `mysql`) are only
|
||||
allowed to login from the database host.
|
||||
|
||||
#### Mailman
|
||||
You should create the following mailing lists from the mail container:
|
||||
|
@ -54,7 +122,7 @@ messages get accepted (by default they get held).
|
|||
#### Dependencies
|
||||
Next, install and activate a virtualenv:
|
||||
```sh
|
||||
sudo apt install libkrb5-dev python3-dev
|
||||
sudo apt install libkrb5-dev libpq-dev python3-dev
|
||||
python3 -m venv venv
|
||||
. venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
|
|
@ -10,7 +10,7 @@ def http_request(method: str, path: str, **kwargs) -> requests.Response:
|
|||
client = component.getUtility(IHTTPClient)
|
||||
cfg = component.getUtility(IConfig)
|
||||
if path.startswith('/api/db'):
|
||||
host = cfg.get('ceod_db_host')
|
||||
host = cfg.get('ceod_database_host')
|
||||
delegate = False
|
||||
else:
|
||||
host = cfg.get('ceod_admin_host')
|
||||
|
|
|
@ -49,3 +49,18 @@ class UserNotSubscribedError(Exception):
|
|||
class NoSuchListError(Exception):
|
||||
def __init__(self):
|
||||
super().__init__('mailing list does not exist')
|
||||
|
||||
|
||||
class InvalidUsernameError(Exception):
|
||||
def __init__(self):
|
||||
super().__init__('Username contains characters that are not allowed')
|
||||
|
||||
|
||||
class DatabaseConnectionError(Exception):
|
||||
def __init__(self):
|
||||
super().__init__('unable to connect or authenticate to sql service')
|
||||
|
||||
|
||||
class DatabasePermissionError(Exception):
|
||||
def __init__(self):
|
||||
super().__init__('unable to perform action due to lack of permissions')
|
||||
|
|
|
@ -0,0 +1,18 @@
|
|||
from zope.interface import Attribute, Interface
|
||||
|
||||
|
||||
class IDatabaseService(Interface):
|
||||
"""Interface to create databases for users."""
|
||||
|
||||
type = Attribute('the type of databases that will be created')
|
||||
auth_username = Attribute('username to a privileged user on the database host')
|
||||
auth_password = Attribute('password to a privileged user on the database host')
|
||||
|
||||
def create_db(username: str) -> str:
|
||||
"""create a user and database and return the password"""
|
||||
|
||||
def reset_passwd(username: str) -> str:
|
||||
"""reset user password and return it"""
|
||||
|
||||
def delete_db(username: str):
|
||||
"""remove user and delete their database"""
|
|
@ -8,3 +8,4 @@ from .IUWLDAPService import IUWLDAPService
|
|||
from .IMailService import IMailService
|
||||
from .IMailmanService import IMailmanService
|
||||
from .IHTTPClient import IHTTPClient
|
||||
from .IDatabaseService import IDatabaseService
|
||||
|
|
|
@ -7,11 +7,12 @@ from zope import component
|
|||
|
||||
from .error_handlers import register_error_handlers
|
||||
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
|
||||
IMailmanService, IMailService, IUWLDAPService, IHTTPClient
|
||||
IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService
|
||||
from ceo_common.model import Config, HTTPClient, RemoteMailmanService
|
||||
from ceod.api.spnego import init_spnego
|
||||
from ceod.model import KerberosService, LDAPService, FileService, \
|
||||
MailmanService, MailService, UWLDAPService
|
||||
from ceod.db import MySQLService, PostgreSQLService
|
||||
|
||||
|
||||
def create_app(flask_config={}):
|
||||
|
@ -36,6 +37,10 @@ def create_app(flask_config={}):
|
|||
from ceod.api import mailman
|
||||
app.register_blueprint(mailman.bp, url_prefix='/api/mailman')
|
||||
|
||||
if hostname == cfg.get('ceod_database_host'):
|
||||
from ceod.api import database
|
||||
app.register_blueprint(database.bp, url_prefix='/api/db')
|
||||
|
||||
from ceod.api import groups
|
||||
app.register_blueprint(groups.bp, url_prefix='/api/groups')
|
||||
|
||||
|
@ -103,3 +108,13 @@ def register_services(app):
|
|||
# UWLDAPService
|
||||
uwldap_srv = UWLDAPService()
|
||||
component.provideUtility(uwldap_srv, IUWLDAPService)
|
||||
|
||||
# MySQLService
|
||||
if hostname == cfg.get('ceod_database_host'):
|
||||
mysql_srv = MySQLService()
|
||||
component.provideUtility(mysql_srv, IDatabaseService, 'mysql')
|
||||
|
||||
# PostgreSQLService
|
||||
if hostname == cfg.get('ceod_database_host'):
|
||||
psql_srv = PostgreSQLService()
|
||||
component.provideUtility(psql_srv, IDatabaseService, 'postgresql')
|
||||
|
|
|
@ -0,0 +1,104 @@
|
|||
from flask import Blueprint
|
||||
from zope import component
|
||||
from functools import wraps
|
||||
|
||||
from ceod.api.utils import authz_restrict_to_syscom, user_is_in_group, \
|
||||
requires_authentication_no_realm, development_only
|
||||
from ceo_common.errors import UserNotFoundError, DatabaseConnectionError, DatabasePermissionError, \
|
||||
InvalidUsernameError, UserAlreadyExistsError
|
||||
from ceo_common.interfaces import ILDAPService, IDatabaseService
|
||||
|
||||
|
||||
bp = Blueprint('db', __name__)
|
||||
|
||||
|
||||
def db_exception_handler(func):
|
||||
@wraps(func)
|
||||
a268wang marked this conversation as resolved
Outdated
merenber
commented
I don't think this is the right type of error to raise here. Should probably be something like 'InvalidUsername'.
I don't think this is the right type of error to raise here. Should probably be something like 'InvalidUsername'.
Also, you want to raise an instance of the Exception class, not the class itself:
```python
raise InvalidUsername()
```
|
||||
def function(db_type: str, username: str):
|
||||
try:
|
||||
# Username should not contain symbols.
|
||||
# Underscores are allowed.
|
||||
for c in username:
|
||||
if not (c.isalnum() or c == '_'):
|
||||
raise InvalidUsernameError()
|
||||
ldap_srv = component.getUtility(ILDAPService)
|
||||
ldap_srv.get_user(username) # make sure user exists
|
||||
return func(db_type, username)
|
||||
except UserNotFoundError:
|
||||
return {'error': 'user not found'}, 404
|
||||
except UserAlreadyExistsError:
|
||||
return {'error': 'database user is already created'}, 409
|
||||
except InvalidUsernameError:
|
||||
return {'error': 'username contains invalid characters'}, 400
|
||||
except DatabaseConnectionError:
|
||||
return {'error': 'unable to connect or authenticate to sql server'}, 500
|
||||
except DatabasePermissionError:
|
||||
return {'error': 'unable to perform action due to permissions'}, 500
|
||||
return function
|
||||
|
||||
|
||||
@db_exception_handler
|
||||
def create_db_from_type(db_type: str, username: str):
|
||||
db_srv = component.getUtility(IDatabaseService, db_type)
|
||||
password = db_srv.create_db(username)
|
||||
return {'password': password}
|
||||
|
||||
|
||||
@db_exception_handler
|
||||
a268wang marked this conversation as resolved
Outdated
merenber
commented
You don't need this last case - unhandled exceptions are handled in error_handlers.py. You don't need this last case - unhandled exceptions are handled in error_handlers.py.
|
||||
def reset_db_passwd_from_type(db_type: str, username: str):
|
||||
db_srv = component.getUtility(IDatabaseService, db_type)
|
||||
password = db_srv.reset_db_passwd(username)
|
||||
return {'password': password}
|
||||
|
||||
|
||||
@db_exception_handler
|
||||
def delete_db_from_type(db_type: str, username: str):
|
||||
db_srv = component.getUtility(IDatabaseService, db_type)
|
||||
db_srv.delete_db(username)
|
||||
return {'status': 'OK'}
|
||||
|
||||
|
||||
@bp.route('/mysql/<username>', methods=['POST'])
|
||||
@requires_authentication_no_realm
|
||||
def create_mysql_db(auth_user: str, username: str):
|
||||
if not (auth_user == username or user_is_in_group(auth_user, 'syscom')):
|
||||
return {'error': "not authorized to create databases for others"}, 403
|
||||
return create_db_from_type('mysql', username)
|
||||
|
||||
|
||||
@bp.route('/postgresql/<username>', methods=['POST'])
|
||||
@requires_authentication_no_realm
|
||||
def create_postgresql_db(auth_user: str, username: str):
|
||||
if not (auth_user == username or user_is_in_group(auth_user, 'syscom')):
|
||||
return {'error': "not authorized to create databases for others"}, 403
|
||||
return create_db_from_type('postgresql', username)
|
||||
|
||||
|
||||
@bp.route('/mysql/<username>/pwreset', methods=['POST'])
|
||||
@requires_authentication_no_realm
|
||||
def reset_mysql_db_passwd(auth_user: str, username: str):
|
||||
if not (auth_user == username or user_is_in_group(auth_user, 'syscom')):
|
||||
return {'error': "not authorized to request password reset for others"}, 403
|
||||
return reset_db_passwd_from_type('mysql', username)
|
||||
|
||||
|
||||
@bp.route('/postgresql/<username>/pwreset', methods=['POST'])
|
||||
@requires_authentication_no_realm
|
||||
def reset_postgresql_db_passwd(auth_user: str, username: str):
|
||||
if not (auth_user == username or user_is_in_group(auth_user, 'syscom')):
|
||||
return {'error': "not authorized to request password reset for others"}, 403
|
||||
return reset_db_passwd_from_type('postgresql', username)
|
||||
|
||||
|
||||
@bp.route('/mysql/<username>', methods=['DELETE'])
|
||||
@authz_restrict_to_syscom
|
||||
@development_only
|
||||
def delete_mysql_db(username: str):
|
||||
return delete_db_from_type('mysql', username)
|
||||
|
||||
|
||||
@bp.route('/postgresql/<username>', methods=['DELETE'])
|
||||
@authz_restrict_to_syscom
|
||||
@development_only
|
||||
def delete_postgresql_db(username: str):
|
||||
return delete_db_from_type('postgresql', username)
|
|
@ -0,0 +1,88 @@
|
|||
from zope.interface import implementer
|
||||
from zope import component
|
||||
from contextlib import contextmanager
|
||||
|
||||
from ceo_common.interfaces import IDatabaseService, IConfig
|
||||
from ceo_common.errors import DatabaseConnectionError, DatabasePermissionError, UserAlreadyExistsError, \
|
||||
UserNotFoundError
|
||||
from ceo_common.logger_factory import logger_factory
|
||||
from ceod.utils import gen_password
|
||||
from ceod.db.utils import response_is_empty
|
||||
|
||||
from mysql.connector import connect
|
||||
from mysql.connector.errors import InterfaceError, ProgrammingError
|
||||
|
||||
a268wang marked this conversation as resolved
Outdated
merenber
commented
I suggest creating a class variable, like so:
I suggest creating a class variable, like so:
```python
class MySQLService:
type = 'mysql'
def __init__(self):
...
```
|
||||
logger = logger_factory(__name__)
|
||||
|
||||
|
||||
@implementer(IDatabaseService)
|
||||
class MySQLService:
|
||||
|
||||
type = 'mysql'
|
||||
|
||||
def __init__(self):
|
||||
a268wang marked this conversation as resolved
Outdated
merenber
commented
I think we should allow CSC members to access their database from any CSC machine. I think we should allow CSC members to access their database from any CSC machine.
|
||||
config = component.getUtility(IConfig)
|
||||
self.auth_username = config.get('mysql_username')
|
||||
self.auth_password = config.get('mysql_password')
|
||||
self.host = config.get('mysql_host')
|
||||
|
||||
@contextmanager
|
||||
def mysql_connection(self):
|
||||
try:
|
||||
with connect(
|
||||
host=self.host,
|
||||
user=self.auth_username,
|
||||
password=self.auth_password,
|
||||
) as con:
|
||||
yield con
|
||||
except InterfaceError as e:
|
||||
logger.error(e)
|
||||
a268wang marked this conversation as resolved
Outdated
merenber
commented
I'm not sure if this is intuitive behaviour - if the user wants to reset their database password, there should be a separate endpoint for that. I'm not sure if this is intuitive behaviour - if the user wants to reset their database password, there should be a separate endpoint for that.
Maybe something like `POST /api/db/mysql/<username>/pwreset`.
merenber
commented
On second thought, I don't think it's necessary to implement password reset behaviour, since the old ceo didn't have it. On second thought, I don't think it's necessary to implement password reset behaviour, since the old ceo didn't have it.
The password will be written to a file in their home directory anyways, so they don't really have an excuse to lose it.
a268wang
commented
The old ceo would reset the database password if you already had a database and tried to create a mysql database. The old ceo would reset the database password if you already had a database and tried to create a mysql database.
I was kind of basing it off that when I wrote this.
merenber
commented
I see. I still think this is non-intuitive behaviour, to be honest. If I clicked a button titled "create database", and it reset my DB password instead, I would be pretty confused. I see. I still think this is non-intuitive behaviour, to be honest. If I clicked a button titled "create database", and it reset my DB password instead, I would be pretty confused.
I think we should just return an error here. If we really want to, we can add an extra endpoint for resetting DB passwords, but this should happen very rarely. Up to you.
a268wang
commented
Ok I'll return a UserAlreadyExistsError (as in mysql/psql user already exists) and add the password reset endpoint since I wrote most of the code already. Ok I'll return a UserAlreadyExistsError (as in mysql/psql user already exists) and add the password reset endpoint since I wrote most of the code already.
|
||||
raise DatabaseConnectionError()
|
||||
except ProgrammingError as e:
|
||||
a268wang marked this conversation as resolved
Outdated
merenber
commented
If the user already exists but somehow their database doesn't, we should return an error. If the user already exists but somehow their database doesn't, we should return an error.
a268wang
commented
I think it would be better if it just did nothing because it is possible for the mysql users to drop their database (and recreate it themselves) But for postgresql if someone somehow dropped their database they wouldn't be able to even login I think it would be better if it just did nothing because it is possible for the mysql users to drop their database (and recreate it themselves)
But for postgresql if someone somehow dropped their database they wouldn't be able to even login
merenber
commented
Sounds good. Sounds good.
|
||||
logger.error(e)
|
||||
raise DatabasePermissionError()
|
||||
|
||||
def create_db(self, username: str) -> str:
|
||||
password = gen_password()
|
||||
search_for_user = f"SELECT user FROM mysql.user WHERE user='{username}'"
|
||||
search_for_db = f"SHOW DATABASES LIKE '{username}'"
|
||||
create_user = f"""
|
||||
CREATE USER '{username}'@'%' IDENTIFIED BY %(password)s;
|
||||
"""
|
||||
create_database = f"""
|
||||
CREATE DATABASE {username};
|
||||
GRANT ALL PRIVILEGES ON {username}.* TO '{username}'@'%';
|
||||
"""
|
||||
|
||||
with self.mysql_connection() as con, con.cursor() as cursor:
|
||||
if response_is_empty(search_for_user, con):
|
||||
cursor.execute(create_user, {'password': password})
|
||||
a268wang marked this conversation as resolved
Outdated
merenber
commented
Not that it matters, but doesn't it make more sense to drop the database before dropping the user? Since the user logically "owns" the database? Not that it matters, but doesn't it make more sense to drop the database before dropping the user? Since the user logically "owns" the database?
|
||||
if response_is_empty(search_for_db, con):
|
||||
cursor.execute(create_database)
|
||||
else:
|
||||
raise UserAlreadyExistsError()
|
||||
return password
|
||||
|
||||
def reset_db_passwd(self, username: str) -> str:
|
||||
password = gen_password()
|
||||
search_for_user = f"SELECT user FROM mysql.user WHERE user='{username}'"
|
||||
reset_password = f"""
|
||||
ALTER USER '{username}'@'%' IDENTIFIED BY %(password)s
|
||||
"""
|
||||
|
||||
with self.mysql_connection() as con, con.cursor() as cursor:
|
||||
if not response_is_empty(search_for_user, con):
|
||||
cursor.execute(reset_password, {'password': password})
|
||||
else:
|
||||
raise UserNotFoundError(username)
|
||||
return password
|
||||
|
||||
def delete_db(self, username: str):
|
||||
drop_db = f"DROP DATABASE IF EXISTS {username}"
|
||||
drop_user = f"""
|
||||
DROP USER IF EXISTS '{username}'@'%';
|
||||
"""
|
||||
|
||||
with self.mysql_connection() as con, con.cursor() as cursor:
|
||||
cursor.execute(drop_db)
|
||||
cursor.execute(drop_user)
|
|
@ -0,0 +1,86 @@
|
|||
from zope.interface import implementer
|
||||
from zope import component
|
||||
from contextlib import contextmanager
|
||||
|
||||
from ceo_common.interfaces import IDatabaseService, IConfig
|
||||
from ceo_common.errors import DatabaseConnectionError, DatabasePermissionError, \
|
||||
UserAlreadyExistsError, UserNotFoundError
|
||||
from ceo_common.logger_factory import logger_factory
|
||||
from ceod.utils import gen_password
|
||||
from ceod.db.utils import response_is_empty
|
||||
|
||||
from psycopg2 import connect, OperationalError, ProgrammingError
|
||||
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
|
||||
|
||||
logger = logger_factory(__name__)
|
||||
|
||||
|
||||
@implementer(IDatabaseService)
|
||||
class PostgreSQLService:
|
||||
|
||||
type = 'postgresql'
|
||||
|
||||
def __init__(self):
|
||||
config = component.getUtility(IConfig)
|
||||
self.auth_username = config.get('postgresql_username')
|
||||
self.auth_password = config.get('postgresql_password')
|
||||
self.host = config.get('postgresql_host')
|
||||
|
||||
@contextmanager
|
||||
def psql_connection(self):
|
||||
con = None
|
||||
try:
|
||||
# Don't use the connection as a context manager, because that
|
||||
# creates a new transaction.
|
||||
con = connect(
|
||||
host=self.host,
|
||||
user=self.auth_username,
|
||||
password=self.auth_password,
|
||||
)
|
||||
con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
|
||||
a268wang marked this conversation as resolved
Outdated
merenber
commented
Same comments as for MySQL. Same comments as for MySQL.
|
||||
yield con
|
||||
except OperationalError as e:
|
||||
logger.error(e)
|
||||
raise DatabaseConnectionError()
|
||||
except ProgrammingError as e:
|
||||
logger.error(e)
|
||||
raise DatabasePermissionError()
|
||||
finally:
|
||||
if con is not None:
|
||||
con.close()
|
||||
|
||||
def create_db(self, username: str) -> str:
|
||||
password = gen_password()
|
||||
search_for_user = f"SELECT FROM pg_roles WHERE rolname='{username}'"
|
||||
search_for_db = f"SELECT FROM pg_database WHERE datname='{username}'"
|
||||
create_user = f"CREATE USER {username} WITH PASSWORD %(password)s"
|
||||
create_database = f"CREATE DATABASE {username} OWNER {username}"
|
||||
revoke_perms = f"REVOKE ALL ON DATABASE {username} FROM PUBLIC"
|
||||
|
||||
with self.psql_connection() as con, con.cursor() as cursor:
|
||||
if not response_is_empty(search_for_user, con):
|
||||
raise UserAlreadyExistsError()
|
||||
cursor.execute(create_user, {'password': password})
|
||||
if response_is_empty(search_for_db, con):
|
||||
cursor.execute(create_database)
|
||||
cursor.execute(revoke_perms)
|
||||
return password
|
||||
|
||||
def reset_db_passwd(self, username: str) -> str:
|
||||
password = gen_password()
|
||||
search_for_user = f"SELECT FROM pg_roles WHERE rolname='{username}'"
|
||||
reset_password = f"ALTER USER {username} WITH PASSWORD %(password)s"
|
||||
|
||||
with self.psql_connection() as con, con.cursor() as cursor:
|
||||
if response_is_empty(search_for_user, con):
|
||||
raise UserNotFoundError(username)
|
||||
cursor.execute(reset_password, {'password': password})
|
||||
return password
|
||||
|
||||
def delete_db(self, username: str):
|
||||
drop_db = f"DROP DATABASE IF EXISTS {username}"
|
||||
drop_user = f"DROP USER IF EXISTS {username}"
|
||||
|
||||
with self.psql_connection() as con, con.cursor() as cursor:
|
||||
cursor.execute(drop_db)
|
||||
cursor.execute(drop_user)
|
|
@ -0,0 +1,2 @@
|
|||
from .MySQLService import MySQLService
|
||||
from .PostgreSQLService import PostgreSQLService
|
|
@ -0,0 +1,5 @@
|
|||
def response_is_empty(query: str, connection) -> bool:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(query)
|
||||
response = cursor.fetchall()
|
||||
return len(response) == 0
|
|
@ -7,3 +7,5 @@ requests==2.26.0
|
|||
requests-gssapi==1.2.3
|
||||
zope.component==5.0.1
|
||||
zope.interface==5.4.0
|
||||
mysql-connector-python==8.0.26
|
||||
psycopg2==2.9.1
|
|
@ -0,0 +1,120 @@
|
|||
import pytest
|
||||
|
||||
from ceod.model import User
|
||||
from mysql.connector import connect
|
||||
from mysql.connector.errors import ProgrammingError
|
||||
|
||||
|
||||
def test_api_create_mysql_db(cfg, client, g_admin_ctx, ldap_user, krb_user):
|
||||
uid = ldap_user.uid
|
||||
|
||||
with g_admin_ctx():
|
||||
user = User(uid='someone_else', cn='Some Name', terms=['s2021'])
|
||||
user.add_to_ldap()
|
||||
|
||||
# user should be able to create db for themselves
|
||||
status, data = client.post(f"/api/db/mysql/{uid}", json={}, principal=uid)
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
passwd = data['password']
|
||||
|
||||
# conflict if attempting to create db when already has one
|
||||
status, data = client.post(f"/api/db/mysql/{uid}", json={}, principal=uid)
|
||||
assert status == 409
|
||||
|
||||
# normal user cannot create db for others
|
||||
status, data = client.post("/api/db/mysql/someone_else", json={}, principal=uid)
|
||||
assert status == 403
|
||||
|
||||
# cannot create db for user not in ldap
|
||||
status, data = client.post("/api/db/mysql/user_not_found", json={})
|
||||
assert status == 404
|
||||
|
||||
# cannot create db when username contains symbols
|
||||
status, data = client.post("/api/db/mysql/!invalid", json={})
|
||||
assert status == 400
|
||||
|
||||
with connect(
|
||||
host=cfg.get('mysql_host'),
|
||||
user=uid,
|
||||
password=passwd,
|
||||
) as con, con.cursor() as cur:
|
||||
cur.execute("SHOW DATABASES")
|
||||
response = cur.fetchall()
|
||||
assert len(response) == 2
|
||||
|
||||
with pytest.raises(ProgrammingError):
|
||||
cur.execute("CREATE DATABASE new_db")
|
||||
|
||||
status, data = client.delete(f"/api/db/mysql/{uid}", json={})
|
||||
assert status == 200
|
||||
|
||||
# user should be deleted
|
||||
with pytest.raises(ProgrammingError):
|
||||
con = connect(
|
||||
host=cfg.get('mysql_host'),
|
||||
user=uid,
|
||||
password=passwd,
|
||||
)
|
||||
|
||||
# db should be deleted
|
||||
with connect(
|
||||
host=cfg.get('mysql_host'),
|
||||
user=cfg.get('mysql_username'),
|
||||
password=cfg.get('mysql_password'),
|
||||
) as con, con.cursor() as cur:
|
||||
cur.execute(f"SHOW DATABASES LIKE '{uid}'")
|
||||
response = cur.fetchall()
|
||||
assert len(response) == 0
|
||||
|
||||
with g_admin_ctx():
|
||||
user.remove_from_ldap()
|
||||
|
||||
|
||||
def test_api_passwd_reset_mysql(cfg, client, g_admin_ctx, ldap_user, krb_user):
|
||||
uid = ldap_user.uid
|
||||
|
||||
with g_admin_ctx():
|
||||
user = User(uid='someone_else', cn='Some Name', terms=['s2021'])
|
||||
user.add_to_ldap()
|
||||
|
||||
status, data = client.post(f"/api/db/mysql/{uid}", json={})
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
old_passwd = data['password']
|
||||
|
||||
con = connect(
|
||||
host=cfg.get('mysql_host'),
|
||||
user=uid,
|
||||
password=old_passwd,
|
||||
)
|
||||
con.close()
|
||||
|
||||
# normal user can get a password reset for themselves
|
||||
status, data = client.post(f"/api/db/mysql/{uid}/pwreset", json={}, principal=uid)
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
new_passwd = data['password']
|
||||
|
||||
assert old_passwd != new_passwd
|
||||
|
||||
# normal user cannot reset password for others
|
||||
status, data = client.post("/api/db/mysql/someone_else/pwreset", json={}, principal=uid)
|
||||
assert status == 403
|
||||
|
||||
# cannot password reset a user that does not have a database
|
||||
status, data = client.post("/api/db/mysql/someone_else/pwreset", json={})
|
||||
assert status == 404
|
||||
|
||||
con = connect(
|
||||
host=cfg.get('mysql_host'),
|
||||
user=uid,
|
||||
password=new_passwd,
|
||||
)
|
||||
con.close()
|
||||
|
||||
status, data = client.delete(f"/api/db/mysql/{uid}", json={})
|
||||
assert status == 200
|
||||
|
||||
with g_admin_ctx():
|
||||
user.remove_from_ldap()
|
|
@ -0,0 +1,123 @@
|
|||
import pytest
|
||||
|
||||
from ceod.model import User
|
||||
from psycopg2 import connect, OperationalError, ProgrammingError
|
||||
|
||||
|
||||
def test_api_create_psql_db(cfg, client, g_admin_ctx, ldap_user, krb_user):
|
||||
uid = ldap_user.uid
|
||||
|
||||
with g_admin_ctx():
|
||||
user = User(uid='someone_else', cn='Some Name', terms=['s2021'])
|
||||
user.add_to_ldap()
|
||||
|
||||
# user should be able to create db for themselves
|
||||
status, data = client.post(f"/api/db/postgresql/{uid}", json={}, principal=uid)
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
passwd = data['password']
|
||||
|
||||
# conflict if attempting to create db when already has one
|
||||
status, data = client.post(f"/api/db/postgresql/{uid}", json={}, principal=uid)
|
||||
assert status == 409
|
||||
|
||||
# normal user cannot create db for others
|
||||
status, data = client.post("/api/db/postgresql/someone_else", json={}, principal=uid)
|
||||
assert status == 403
|
||||
|
||||
# cannot create db for user not in ldap
|
||||
status, data = client.post("/api/db/postgresql/user_not_found", json={})
|
||||
assert status == 404
|
||||
|
||||
# cannot create db when username contains symbols
|
||||
status, data = client.post("/api/db/postgresql/!invalid", json={})
|
||||
assert status == 400
|
||||
|
||||
con = connect(
|
||||
host=cfg.get('postgresql_host'),
|
||||
user=uid,
|
||||
password=passwd,
|
||||
)
|
||||
con.autocommit = True
|
||||
with con.cursor() as cur:
|
||||
cur.execute("SELECT datname FROM pg_database")
|
||||
response = cur.fetchall()
|
||||
# 3 of the 4 are postgres, template0, template1
|
||||
assert len(response) == 4
|
||||
with pytest.raises(ProgrammingError):
|
||||
cur.execute("CREATE DATABASE new_db")
|
||||
con.close()
|
||||
|
||||
status, data = client.delete(f"/api/db/postgresql/{uid}", json={})
|
||||
assert status == 200
|
||||
|
||||
# user should be deleted
|
||||
with pytest.raises(OperationalError):
|
||||
con = connect(
|
||||
host=cfg.get('postgresql_host'),
|
||||
user=uid,
|
||||
password=passwd,
|
||||
)
|
||||
|
||||
# db should be deleted
|
||||
with connect(
|
||||
host=cfg.get('postgresql_host'),
|
||||
user=cfg.get('postgresql_username'),
|
||||
password=cfg.get('postgresql_password'),
|
||||
) as con, con.cursor() as cur:
|
||||
cur.execute(f"SELECT datname FROM pg_database WHERE datname = '{uid}'")
|
||||
response = cur.fetchall()
|
||||
assert len(response) == 0
|
||||
|
||||
with g_admin_ctx():
|
||||
user.remove_from_ldap()
|
||||
|
||||
|
||||
def test_api_passwd_reset_psql(cfg, client, g_admin_ctx, ldap_user, krb_user):
|
||||
uid = ldap_user.uid
|
||||
|
||||
with g_admin_ctx():
|
||||
user = User(uid='someone_else', cn='Some Name', terms=['s2021'])
|
||||
user.add_to_ldap()
|
||||
|
||||
status, data = client.post(f"/api/db/postgresql/{uid}", json={})
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
old_passwd = data['password']
|
||||
|
||||
con = connect(
|
||||
host=cfg.get('postgresql_host'),
|
||||
user=uid,
|
||||
password=old_passwd,
|
||||
)
|
||||
con.close()
|
||||
|
||||
# normal user can get a password reset for themselves
|
||||
status, data = client.post(f"/api/db/postgresql/{uid}/pwreset", json={}, principal=uid)
|
||||
assert status == 200
|
||||
assert 'password' in data
|
||||
new_passwd = data['password']
|
||||
|
||||
assert old_passwd != new_passwd
|
||||
|
||||
# normal user cannot reset password for others
|
||||
status, data = client.post("/api/db/postgresql/someone_else/pwreset",
|
||||
json={}, principal=uid)
|
||||
assert status == 403
|
||||
|
||||
# cannot password reset a user that does not have a database
|
||||
status, data = client.post("/api/db/postgresql/someone_else/pwreset", json={})
|
||||
assert status == 404
|
||||
|
||||
con = connect(
|
||||
host=cfg.get('postgresql_host'),
|
||||
user=uid,
|
||||
password=new_passwd,
|
||||
)
|
||||
con.close()
|
||||
|
||||
status, data = client.delete(f"/api/db/postgresql/{uid}", json={})
|
||||
assert status == 200
|
||||
|
||||
with g_admin_ctx():
|
||||
user.remove_from_ldap()
|
|
@ -7,6 +7,7 @@ admin_host = phosphoric-acid
|
|||
# this is the host with NFS no_root_squash
|
||||
fs_root_host = phosphoric-acid
|
||||
mailman_host = mail
|
||||
database_host = coffee
|
||||
use_https = false
|
||||
port = 9987
|
||||
|
||||
|
@ -56,3 +57,13 @@ exec = exec
|
|||
required = president,vice-president,sysadmin
|
||||
available = president,vice-president,treasurer,secretary,
|
||||
sysadmin,cro,librarian,imapd,webmaster,offsck
|
||||
|
||||
[mysql]
|
||||
username = mysql
|
||||
password = mysql
|
||||
host = localhost
|
||||
|
||||
[postgresql]
|
||||
username = postgres
|
||||
password = postgres
|
||||
host = localhost
|
||||
|
|
|
@ -7,6 +7,7 @@ uw_domain = uwaterloo.internal
|
|||
admin_host = phosphoric-acid
|
||||
fs_root_host = phosphoric-acid
|
||||
mailman_host = phosphoric-acid
|
||||
database_host = phosphoric-acid
|
||||
use_https = false
|
||||
port = 9987
|
||||
|
||||
|
@ -55,3 +56,13 @@ exec = exec
|
|||
required = president,vice-president,sysadmin
|
||||
available = president,vice-president,treasurer,secretary,
|
||||
sysadmin,cro,librarian,imapd,webmaster,offsck
|
||||
|
||||
[mysql]
|
||||
username = mysql
|
||||
password = mysql
|
||||
host = coffee
|
||||
|
||||
[postgresql]
|
||||
username = postgres
|
||||
password = postgres
|
||||
host = coffee
|
||||
|
|
|
@ -6,6 +6,7 @@ import os
|
|||
import pwd
|
||||
import shutil
|
||||
import subprocess
|
||||
from subprocess import DEVNULL
|
||||
import sys
|
||||
import time
|
||||
from unittest.mock import patch, Mock
|
||||
|
@ -20,9 +21,11 @@ from zope import component
|
|||
|
||||
from .utils import gssapi_token_ctx, ccache_cleanup # noqa: F401
|
||||
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
|
||||
IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService
|
||||
IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
|
||||
IDatabaseService
|
||||
from ceo_common.model import Config, HTTPClient
|
||||
from ceod.api import create_app
|
||||
from ceod.db import MySQLService, PostgreSQLService
|
||||
from ceod.model import KerberosService, LDAPService, FileService, User, \
|
||||
MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService
|
||||
import ceod.utils as utils
|
||||
|
@ -241,6 +244,20 @@ def mail_srv(cfg, mock_mail_server):
|
|||
return _mail_srv
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def mysql_srv(cfg):
|
||||
mysql_srv = MySQLService()
|
||||
component.provideUtility(mysql_srv, IDatabaseService, 'mysql')
|
||||
return mysql_srv
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def postgresql_srv(cfg):
|
||||
psql_srv = PostgreSQLService()
|
||||
component.provideUtility(psql_srv, IDatabaseService, 'postgresql')
|
||||
return psql_srv
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope='session')
|
||||
def app(
|
||||
cfg,
|
||||
|
@ -250,6 +267,8 @@ def app(
|
|||
mailman_srv,
|
||||
uwldap_srv,
|
||||
mail_srv,
|
||||
mysql_srv,
|
||||
postgresql_srv,
|
||||
):
|
||||
app = create_app({'TESTING': True})
|
||||
return app
|
||||
|
@ -299,7 +318,11 @@ def ldap_user(simple_user, g_admin_ctx):
|
|||
|
||||
@pytest.fixture
|
||||
def krb_user(simple_user):
|
||||
simple_user.add_to_kerberos('krb5')
|
||||
# We don't want to use add_to_kerberos() here because that expires the
|
||||
# user's password, which we don't want for testing
|
||||
subprocess.run(
|
||||
['kadmin', '-k', '-p', 'ceod/admin', 'addprinc', '-pw', 'krb5',
|
||||
simple_user.uid], stdout=DEVNULL, check=True)
|
||||
yield simple_user
|
||||
simple_user.remove_from_kerberos()
|
||||
|
||||
|
|
I think it would be useful for CSC members to access their PostgreSQL database from any CSC machine. (We should also do the same for MySQL at some point).
Will it be fine if I just change the postgres config to allow connections from any host?
I think so.
For MySQL, I think it should be possible to allow connections for users from a specific subnet: https://stackoverflow.com/questions/11742963/how-to-grant-remote-access-to-mysql-for-a-whole-subnet/38389851#38389851
I've changed both mysql and postgres to allow connections from any source.
Is there a subnet that you want me to use instead?
129.97.134.0/24, 2620:101:f000:4901::/64 (MC VLAN 134)
This should be configurable from ceod.ini.
For the dev environment, use 192.168.100.0/24.
Hmm now that I look into it I think filtering by subnet is a bit iffy.
pg_hba.conf
192.168.100.%
or192.168.100.0/255.255.255.0
)127.0.0.1
(overwise MySQL will try to create the same user twice)Alright, in that case, don't bother. Just let members connect from any IP address.