Add database CLI (#15)

Closes #12

Co-authored-by: Andrew Wang <a268wang@csclub.uwaterloo.ca>
Co-authored-by: Max Erenberg <merenber@csclub.uwaterloo.ca>
Reviewed-on: #15
Co-authored-by: Andrew Wang <a268wang@localhost>
Co-committed-by: Andrew Wang <a268wang@localhost>
pull/20/head
Andrew Wang 1 year ago committed by Max Erenberg
parent cb6243c3e2
commit 33323fd112
  1. 2
      .drone/coffee-setup.sh
  2. 2
      README.md
  3. 104
      ceo/cli/database.py
  4. 4
      ceo/cli/entrypoint.py
  5. 26
      ceo/cli/mysql.py
  6. 26
      ceo/cli/postgresql.py
  7. 11
      ceo/cli/utils.py
  8. 4
      ceod/api/database.py
  9. 30
      ceod/db/MySQLService.py
  10. 2
      ceod/db/PostgreSQLService.py
  11. 13
      ceod/model/User.py
  12. 82
      tests/ceo/cli/test_db_mysql.py
  13. 84
      tests/ceo/cli/test_db_postgresql.py
  14. 7
      tests/ceo_dev.ini
  15. 2
      tests/ceod_dev.ini

@ -30,7 +30,7 @@ host all postgres 0.0.0.0/0 md5
local all all peer
host all all localhost md5
local sameuser all md5
local sameuser all peer
host sameuser all 0.0.0.0/0 md5
EOF
grep -Eq "^listen_addresses = '*'$" $POSTGRES_DIR/postgresql.conf || \

@ -81,7 +81,7 @@ host all postgres 0.0.0.0/0 md5
local all all peer
host all all localhost md5
local sameuser all md5
local sameuser all peer
host sameuser all 0.0.0.0/0 md5
```
**Warning**: in prod, the postgres user should only be allowed to connect locally,

@ -0,0 +1,104 @@
import os
from typing import Dict
import click
from zope import component
from ..utils import http_post, http_get, http_delete
from .utils import handle_sync_response, check_file_path, check_if_in_development
from ceo_common.interfaces import IConfig
def db_cli_response(filename: str, user_dict: Dict, password: str, db_type: str, op: str):
cfg_srv = component.getUtility(IConfig)
db_host = cfg_srv.get(f'{db_type}_host')
username = user_dict['uid']
if db_type == 'mysql':
db_type_name = 'MySQL'
db_cli_local_cmd = f'mysql {username}'
db_cli_cmd = f'mysql {username} -h {db_host} -u {username} -p'
else:
db_type_name = 'PostgreSQL'
db_cli_local_cmd = f'psql {username}'
db_cli_cmd = f'psql -d {username} -h {db_host} -U {username} -W'
username = user_dict['uid']
info = f"""{db_type_name} Database Information for {username}
Your new {db_type_name} database was created. To connect, use the following options:
Database: {username}
Username: {username}
Password: {password}
Host: {db_host}
On {db_host} to connect using the {db_type_name} command-line client use
{db_cli_local_cmd}
From other CSC machines you can connect using
{db_cli_cmd}
"""
wrote_to_file = False
try:
# TODO: use phosphoric-acid to write to file (phosphoric-acid makes
# internal API call to caffeine)
with click.open_file(filename, "w") as f:
f.write(info)
os.chown(filename, user_dict['uid_number'], user_dict['gid_number'])
os.chmod(filename, 0o640)
wrote_to_file = True
except PermissionError:
pass
if op == 'create':
click.echo(f'{db_type_name} database created.')
click.echo(f'''Connection Information:
Database: {username}
Username: {username}
Password: {password}
Host: {db_host}''')
if wrote_to_file:
click.echo(f"\nThese settings have been written to {filename}.")
else:
click.echo(f"\nWe were unable to write these settings to {filename}.")
def create(username: str, db_type: str):
db_type_name = 'MySQL' if db_type == 'mysql' else 'PostgreSQL'
resp = http_get(f'/api/members/{username}')
user_dict = handle_sync_response(resp)
click.confirm(f'Are you sure you want to create a {db_type_name} database for {username}?', abort=True)
info_file_path = os.path.join(user_dict['home_directory'], f"ceo-{db_type}-info")
check_file_path(info_file_path)
resp = http_post(f'/api/db/{db_type}/{username}')
result = handle_sync_response(resp)
password = result['password']
db_cli_response(info_file_path, user_dict, password, db_type, 'create')
def pwreset(username: str, db_type: str):
db_type_name = 'MySQL' if db_type == 'mysql' else 'PostgreSQL'
resp = http_get(f'/api/members/{username}')
user_dict = handle_sync_response(resp)
click.confirm(f'Are you sure you want reset the {db_type_name} password for {username}?', abort=True)
info_file_path = os.path.join(user_dict['home_directory'], f"ceo-{db_type}-info")
check_file_path(info_file_path)
resp = http_post(f'/api/db/{db_type}/{username}/pwreset')
result = handle_sync_response(resp)
password = result['password']
db_cli_response(info_file_path, user_dict, password, db_type, 'pwreset')
def delete(username: str, db_type: str):
check_if_in_development()
db_type_name = 'MySQL' if db_type == 'mysql' else 'PostgreSQL'
click.confirm(f"Are you sure you want to delete the {db_type_name} database for {username}?", abort=True)
resp = http_delete(f'/api/db/{db_type}/{username}')
handle_sync_response(resp)

@ -4,6 +4,8 @@ from .members import members
from .groups import groups
from .positions import positions
from .updateprograms import updateprograms
from .mysql import mysql
from .postgresql import postgresql
@click.group()
@ -15,3 +17,5 @@ cli.add_command(members)
cli.add_command(groups)
cli.add_command(positions)
cli.add_command(updateprograms)
cli.add_command(mysql)
cli.add_command(postgresql)

@ -0,0 +1,26 @@
import click
from .database import create as db_create, pwreset as db_pwreset, delete as db_delete
@click.group(short_help='Perform operations on MySQL')
def mysql():
pass
@mysql.command(short_help='Create a MySQL database for a user')
@click.argument('username')
def create(username):
db_create(username, 'mysql')
@mysql.command(short_help='Reset the password of a MySQL user')
@click.argument('username')
def pwreset(username):
db_pwreset(username, 'mysql')
@mysql.command(short_help="Delete the database of a MySQL user")
@click.argument('username')
def delete(username):
db_delete(username, 'mysql')

@ -0,0 +1,26 @@
import click
from .database import create as db_create, pwreset as db_pwreset, delete as db_delete
@click.group(short_help='Perform operations on PostgreSQL')
def postgresql():
pass
@postgresql.command(short_help='Create a PostgreSQL database for a user')
@click.argument('username')
def create(username):
db_create(username, 'postgresql')
@postgresql.command(short_help='Reset the password of a PostgreSQL user')
@click.argument('username')
def pwreset(username):
db_pwreset(username, 'postgresql')
@postgresql.command(short_help="Delete the database of a PostgreSQL user")
@click.argument('username')
def delete(username):
db_delete(username, 'postgresql')

@ -1,4 +1,6 @@
import socket
import os
from typing import List, Tuple, Dict
import click
@ -50,6 +52,15 @@ def handle_sync_response(resp: requests.Response):
return resp.json()
def check_file_path(file):
if os.path.isfile(file):
click.echo(f"{file} will be overwritten")
click.confirm('Do you want to continue?', abort=True)
elif os.path.isdir(file):
click.echo(f"Error: there exists a directory at {file}")
raise Abort()
def check_if_in_development() -> bool:
"""Aborts if we are not currently in the dev environment."""
if not socket.getfqdn().endswith('.csclub.internal'):

@ -31,9 +31,9 @@ def db_exception_handler(func):
except InvalidUsernameError:
return {'error': 'username contains invalid characters'}, 400
except DatabaseConnectionError:
return {'error': 'unable to connect or authenticate to sql server'}, 500
return {'error': 'unable to connect to sql server'}, 500
except DatabasePermissionError:
return {'error': 'unable to perform action due to permissions'}, 500
return {'error': 'unable to connect or action failed due to permissions'}, 500
return function

@ -10,7 +10,7 @@ from ceod.utils import gen_password
from ceod.db.utils import response_is_empty
from mysql.connector import connect
from mysql.connector.errors import InterfaceError, ProgrammingError
from mysql.connector.errors import OperationalError, ProgrammingError
logger = logger_factory(__name__)
@ -35,9 +35,11 @@ class MySQLService:
password=self.auth_password,
) as con:
yield con
except InterfaceError as e:
# unable to connect
except OperationalError as e:
logger.error(e)
raise DatabaseConnectionError()
# invalid credentials / user does not exist / invalid permissions for action
except ProgrammingError as e:
logger.error(e)
raise DatabasePermissionError()
@ -47,24 +49,21 @@ class MySQLService:
search_for_user = f"SELECT user FROM mysql.user WHERE user='{username}'"
search_for_db = f"SHOW DATABASES LIKE '{username}'"
# CREATE USER can't be used in a query with multiple statements
create_user_commands = [
f"CREATE USER '{username}'@'localhost' IDENTIFIED BY %(password)s",
f"CREATE USER '{username}'@'%' IDENTIFIED BY %(password)s",
]
create_local_user_cmd = f"CREATE USER '{username}'@'localhost' IDENTIFIED VIA unix_socket"
create_user_cmd = f"CREATE USER '{username}'@'%' IDENTIFIED BY %(password)s"
create_database = f"""
CREATE DATABASE {username};
GRANT ALL PRIVILEGES ON {username}.* TO '{username}'@'localhost';
GRANT ALL PRIVILEGES ON {username}.* TO '{username}'@'localhost' IDENTIFIED VIA unix_socket;
GRANT ALL PRIVILEGES ON {username}.* TO '{username}'@'%';
"""
with self.mysql_connection() as con, con.cursor() as cursor:
if response_is_empty(search_for_user, con):
for cmd in create_user_commands:
cursor.execute(cmd, {'password': password})
if response_is_empty(search_for_db, con):
cursor.execute(create_database)
else:
if not response_is_empty(search_for_user, con):
raise UserAlreadyExistsError()
cursor.execute(create_local_user_cmd)
cursor.execute(create_user_cmd, {'password': password})
if response_is_empty(search_for_db, con):
cursor.execute(create_database)
return password
def reset_db_passwd(self, username: str) -> str:
@ -76,10 +75,9 @@ class MySQLService:
"""
with self.mysql_connection() as con, con.cursor() as cursor:
if not response_is_empty(search_for_user, con):
cursor.execute(reset_password, {'password': password})
else:
if response_is_empty(search_for_user, con):
raise UserNotFoundError(username)
cursor.execute(reset_password, {'password': password})
return password
def delete_db(self, username: str):

@ -39,9 +39,11 @@ class PostgreSQLService:
)
con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
yield con
# unable to connect / invalid credentials / user does not exist
except OperationalError as e:
logger.error(e)
raise DatabaseConnectionError()
# invalid permissions for action
except ProgrammingError as e:
logger.error(e)
raise DatabasePermissionError()

@ -56,7 +56,6 @@ class User:
self.ldap_srv = component.getUtility(ILDAPService)
self.krb_srv = component.getUtility(IKerberosService)
self.file_srv = component.getUtility(IFileService)
def to_dict(self, get_forwarding_addresses: bool = False) -> Dict:
data = {
@ -103,10 +102,12 @@ class User:
self.krb_srv.change_password(self.uid, password)
def create_home_dir(self):
self.file_srv.create_home_dir(self)
file_srv = component.getUtility(IFileService)
file_srv.create_home_dir(self)
def delete_home_dir(self):
self.file_srv.delete_home_dir(self)
file_srv = component.getUtility(IFileService)
file_srv.delete_home_dir(self)
def subscribe_to_mailing_list(self, mailing_list: str):
component.getUtility(IMailmanService).subscribe(self.uid, mailing_list)
@ -163,7 +164,9 @@ class User:
self.positions = positions
def get_forwarding_addresses(self) -> List[str]:
return self.file_srv.get_forwarding_addresses(self)
file_srv = component.getUtility(IFileService)
return file_srv.get_forwarding_addresses(self)
def set_forwarding_addresses(self, addresses: List[str]):
self.file_srv.set_forwarding_addresses(self, addresses)
file_srv = component.getUtility(IFileService)
file_srv.set_forwarding_addresses(self, addresses)

@ -0,0 +1,82 @@
import os
from click.testing import CliRunner
from mysql.connector import connect
from mysql.connector.errors import ProgrammingError
import pytest
from ceo.cli import cli
def mysql_attempt_connection(host, username, password):
with connect(
host=host,
user=username,
password=password,
) as con, con.cursor() as cur:
cur.execute("SHOW DATABASES")
response = cur.fetchall()
assert len(response) == 2
with pytest.raises(ProgrammingError):
cur.execute("CREATE DATABASE new_db")
def test_mysql(cli_setup, cfg, ldap_user):
runner = CliRunner()
username = ldap_user.uid
os.makedirs(ldap_user.home_directory)
host = cfg.get("mysql_host")
info_file_path = os.path.join(ldap_user.home_directory, "ceo-mysql-info")
assert not os.path.isfile(info_file_path)
# create database for user
result = runner.invoke(cli, ['mysql', 'create', username], input='y\n')
assert result.exit_code == 0
assert os.path.isfile(info_file_path)
response_arr = result.output.split()
passwd = response_arr[response_arr.index("Password:") + 1]
with open(info_file_path, 'r') as file:
old_info = file.read()
expected = f"""Are you sure you want to create a MySQL database for {username}? [y/N]: y
MySQL database created.
Connection Information:
Database: {username}
Username: {username}
Password: {passwd}
Host: {host}
These settings have been written to {info_file_path}.
"""
assert result.output == expected
mysql_attempt_connection(host, username, passwd)
# perform password reset for user
# confirm once to reset password, another to overwrite the file
result = runner.invoke(cli, ['mysql', 'pwreset', username], input="y\ny\n")
assert result.exit_code == 0
response_arr = result.output.split()
new_passwd = response_arr[response_arr.index("Password:") + 1]
with open(info_file_path, 'r') as file:
new_info = file.read()
assert new_passwd != passwd
assert old_info != new_info
mysql_attempt_connection(host, username, new_passwd)
# delete database and file
result = runner.invoke(cli, ['mysql', 'delete', username], input="y\n")
assert result.exit_code == 0
# user should be deleted
with pytest.raises(ProgrammingError):
mysql_attempt_connection(host, username, passwd)
os.remove(info_file_path)
os.rmdir(ldap_user.home_directory)

@ -0,0 +1,84 @@
import pytest
import os
from click.testing import CliRunner
from ceo.cli import cli
from psycopg2 import connect, OperationalError, ProgrammingError
def psql_attempt_connection(host, username, password):
con = connect(
host=host,
user=username,
password=password,
)
con.autocommit = True
with con.cursor() as cur:
cur.execute("SELECT datname FROM pg_database")
response = cur.fetchall()
# 3 of the 4 are postgres, template0, template1
assert len(response) == 4
with pytest.raises(ProgrammingError):
cur.execute("CREATE DATABASE new_db")
con.close()
def test_postgresql(cli_setup, cfg, ldap_user):
runner = CliRunner()
username = ldap_user.uid
os.makedirs(ldap_user.home_directory)
host = cfg.get("postgresql_host")
info_file_path = os.path.join(ldap_user.home_directory, "ceo-postgresql-info")
assert not os.path.isfile(info_file_path)
# create database for user
result = runner.invoke(cli, ['postgresql', 'create', username], input='y\n')
assert result.exit_code == 0
assert os.path.isfile(info_file_path)
response_arr = result.output.split()
passwd = response_arr[response_arr.index("Password:") + 1]
with open(info_file_path, 'r') as file:
old_info = file.read()
expected = f"""Are you sure you want to create a PostgreSQL database for {username}? [y/N]: y
PostgreSQL database created.
Connection Information:
Database: {username}
Username: {username}
Password: {passwd}
Host: {host}
These settings have been written to {info_file_path}.
"""
assert result.output == expected
psql_attempt_connection(host, username, passwd)
# perform password reset for user
# confirm once to reset password, another to overwrite the file
result = runner.invoke(cli, ['postgresql', 'pwreset', username], input="y\ny\n")
assert result.exit_code == 0
response_arr = result.output.split()
new_passwd = response_arr[response_arr.index("Password:") + 1]
with open(info_file_path, 'r') as file:
new_info = file.read()
assert new_passwd != passwd
assert old_info != new_info
psql_attempt_connection(host, username, new_passwd)
# delete database and file
result = runner.invoke(cli, ['postgresql', 'delete', username], input="y\n")
assert result.exit_code == 0
# user should be deleted
with pytest.raises(OperationalError):
psql_attempt_connection(host, username, passwd)
os.remove(info_file_path)
os.rmdir(ldap_user.home_directory)

@ -5,6 +5,7 @@ uw_domain = uwaterloo.internal
[ceod]
# this is the host with the ceod/admin Kerberos key
admin_host = phosphoric-acid
database_host = coffee
use_https = false
port = 9987
@ -12,3 +13,9 @@ port = 9987
required = president,vice-president,sysadmin
available = president,vice-president,treasurer,secretary,
sysadmin,cro,librarian,imapd,webmaster,offsck
[mysql]
host = coffee
[postgresql]
host = coffee

@ -13,7 +13,7 @@ port = 9987
[ldap]
admin_principal = ceod/admin
server_url = ldap://ldap-master.csclub.internal
server_url = ldap://auth1.csclub.internal
sasl_realm = CSCLUB.INTERNAL
users_base = ou=People,dc=csclub,dc=internal
groups_base = ou=Group,dc=csclub,dc=internal

Loading…
Cancel
Save