Add database CLI (#15)

Closes #12

Co-authored-by: Andrew Wang <a268wang@csclub.uwaterloo.ca>
Co-authored-by: Max Erenberg <merenber@csclub.uwaterloo.ca>
Reviewed-on: #15
Co-authored-by: Andrew Wang <a268wang@localhost>
Co-committed-by: Andrew Wang <a268wang@localhost>

.drone/coffee-setup.sh

@@ -30,7 +30,7 @@ host    all             postgres        0.0.0.0/0               md5
 local   all             all                                     peer
 host    all             all             localhost               md5
 
-local   sameuser        all                                     md5
+local   sameuser        all                                     peer
 host    sameuser        all             0.0.0.0/0               md5
 EOF
 grep -Eq "^listen_addresses = '*'$" $POSTGRES_DIR/postgresql.conf || \

README.md

@@ -81,7 +81,7 @@ host    all             postgres        0.0.0.0/0               md5
 local   all             all                                     peer
 host    all             all             localhost               md5
 
-local   sameuser        all                                     md5
+local   sameuser        all                                     peer
 host    sameuser        all             0.0.0.0/0               md5
 ```
 **Warning**: in prod, the postgres user should only be allowed to connect locally,

ceo/cli/database.py

@@ -0,0 +1,104 @@
+import os
+from typing import Dict
+
+import click
+from zope import component
+
+from ..utils import http_post, http_get, http_delete
+from .utils import handle_sync_response, check_file_path, check_if_in_development
+from ceo_common.interfaces import IConfig
+
+
+def db_cli_response(filename: str, user_dict: Dict, password: str, db_type: str, op: str):
+    cfg_srv = component.getUtility(IConfig)
+    db_host = cfg_srv.get(f'{db_type}_host')
+    username = user_dict['uid']
+    if db_type == 'mysql':
+        db_type_name = 'MySQL'
+        db_cli_local_cmd = f'mysql {username}'
+        db_cli_cmd = f'mysql {username} -h {db_host} -u {username} -p'
+    else:
+        db_type_name = 'PostgreSQL'
+        db_cli_local_cmd = f'psql {username}'
+        db_cli_cmd = f'psql -d {username} -h {db_host} -U {username} -W'
+    username = user_dict['uid']
+    info = f"""{db_type_name} Database Information for {username}
+
+Your new {db_type_name} database was created. To connect, use the following options:
+
+Database: {username}
+Username: {username}
+Password: {password}
+Host: {db_host}
+
+On {db_host} to connect using the {db_type_name} command-line client use
+
+    {db_cli_local_cmd}
+
+From other CSC machines you can connect using
+
+    {db_cli_cmd}
+"""
+    wrote_to_file = False
+    try:
+        # TODO: use phosphoric-acid to write to file (phosphoric-acid makes
+        # internal API call to caffeine)
+        with click.open_file(filename, "w") as f:
+            f.write(info)
+        os.chown(filename, user_dict['uid_number'], user_dict['gid_number'])
+        os.chmod(filename, 0o640)
+        wrote_to_file = True
+    except PermissionError:
+        pass
+    if op == 'create':
+        click.echo(f'{db_type_name} database created.')
+    click.echo(f'''Connection Information:
+
+Database: {username}
+Username: {username}
+Password: {password}
+Host: {db_host}''')
+    if wrote_to_file:
+        click.echo(f"\nThese settings have been written to {filename}.")
+    else:
+        click.echo(f"\nWe were unable to write these settings to {filename}.")
+
+
+def create(username: str, db_type: str):
+    db_type_name = 'MySQL' if db_type == 'mysql' else 'PostgreSQL'
+    resp = http_get(f'/api/members/{username}')
+    user_dict = handle_sync_response(resp)
+    click.confirm(f'Are you sure you want to create a {db_type_name} database for {username}?', abort=True)
+
+    info_file_path = os.path.join(user_dict['home_directory'], f"ceo-{db_type}-info")
+    check_file_path(info_file_path)
+
+    resp = http_post(f'/api/db/{db_type}/{username}')
+    result = handle_sync_response(resp)
+    password = result['password']
+
+    db_cli_response(info_file_path, user_dict, password, db_type, 'create')
+
+
+def pwreset(username: str, db_type: str):
+    db_type_name = 'MySQL' if db_type == 'mysql' else 'PostgreSQL'
+    resp = http_get(f'/api/members/{username}')
+    user_dict = handle_sync_response(resp)
+    click.confirm(f'Are you sure you want reset the {db_type_name} password for {username}?', abort=True)
+
+    info_file_path = os.path.join(user_dict['home_directory'], f"ceo-{db_type}-info")
+    check_file_path(info_file_path)
+
+    resp = http_post(f'/api/db/{db_type}/{username}/pwreset')
+    result = handle_sync_response(resp)
+    password = result['password']
+
+    db_cli_response(info_file_path, user_dict, password, db_type, 'pwreset')
+
+
+def delete(username: str, db_type: str):
+    check_if_in_development()
+    db_type_name = 'MySQL' if db_type == 'mysql' else 'PostgreSQL'
+    click.confirm(f"Are you sure you want to delete the {db_type_name} database for {username}?", abort=True)
+    resp = http_delete(f'/api/db/{db_type}/{username}')
+    handle_sync_response(resp)

ceo/cli/entrypoint.py

@@ -4,6 +4,8 @@ from .members import members
 from .groups import groups
 from .positions import positions
 from .updateprograms import updateprograms
+from .mysql import mysql
+from .postgresql import postgresql
 
 
 @click.group()
@@ -15,3 +17,5 @@ cli.add_command(members)
 cli.add_command(groups)
 cli.add_command(positions)
 cli.add_command(updateprograms)
+cli.add_command(mysql)
+cli.add_command(postgresql)

ceo/cli/mysql.py

@@ -0,0 +1,26 @@
+import click
+
+from .database import create as db_create, pwreset as db_pwreset, delete as db_delete
+
+
+@click.group(short_help='Perform operations on MySQL')
+def mysql():
+    pass
+
+
+@mysql.command(short_help='Create a MySQL database for a user')
+@click.argument('username')
+def create(username):
+    db_create(username, 'mysql')
+
+
+@mysql.command(short_help='Reset the password of a MySQL user')
+@click.argument('username')
+def pwreset(username):
+    db_pwreset(username, 'mysql')
+
+
+@mysql.command(short_help="Delete the database of a MySQL user")
+@click.argument('username')
+def delete(username):
+    db_delete(username, 'mysql')

ceo/cli/postgresql.py

@@ -0,0 +1,26 @@
+import click
+
+from .database import create as db_create, pwreset as db_pwreset, delete as db_delete
+
+
+@click.group(short_help='Perform operations on PostgreSQL')
+def postgresql():
+    pass
+
+
+@postgresql.command(short_help='Create a PostgreSQL database for a user')
+@click.argument('username')
+def create(username):
+    db_create(username, 'postgresql')
+
+
+@postgresql.command(short_help='Reset the password of a PostgreSQL user')
+@click.argument('username')
+def pwreset(username):
+    db_pwreset(username, 'postgresql')
+
+
+@postgresql.command(short_help="Delete the database of a PostgreSQL user")
+@click.argument('username')
+def delete(username):
+    db_delete(username, 'postgresql')

ceo/cli/utils.py

@@ -1,4 +1,6 @@
 import socket
+import os
+
 from typing import List, Tuple, Dict
 
 import click
@@ -50,6 +52,15 @@ def handle_sync_response(resp: requests.Response):
     return resp.json()
 
 
+def check_file_path(file):
+    if os.path.isfile(file):
+        click.echo(f"{file} will be overwritten")
+        click.confirm('Do you want to continue?', abort=True)
+    elif os.path.isdir(file):
+        click.echo(f"Error: there exists a directory at {file}")
+        raise Abort()
+
+
 def check_if_in_development() -> bool:
     """Aborts if we are not currently in the dev environment."""
     if not socket.getfqdn().endswith('.csclub.internal'):

ceod/api/database.py

@@ -31,9 +31,9 @@ def db_exception_handler(func):
         except InvalidUsernameError:
             return {'error': 'username contains invalid characters'}, 400
         except DatabaseConnectionError:
-            return {'error': 'unable to connect or authenticate to sql server'}, 500
+            return {'error': 'unable to connect to sql server'}, 500
         except DatabasePermissionError:
-            return {'error': 'unable to perform action due to permissions'}, 500
+            return {'error': 'unable to connect or action failed due to permissions'}, 500
     return function
 
 

ceod/db/MySQLService.py

@@ -10,7 +10,7 @@ from ceod.utils import gen_password
 from ceod.db.utils import response_is_empty
 
 from mysql.connector import connect
-from mysql.connector.errors import InterfaceError, ProgrammingError
+from mysql.connector.errors import OperationalError, ProgrammingError
 
 logger = logger_factory(__name__)
 
@@ -35,9 +35,11 @@ class MySQLService:
                     password=self.auth_password,
             ) as con:
                 yield con
-        except InterfaceError as e:
+        # unable to connect
+        except OperationalError as e:
             logger.error(e)
             raise DatabaseConnectionError()
+        # invalid credentials / user does not exist / invalid permissions for action
         except ProgrammingError as e:
             logger.error(e)
             raise DatabasePermissionError()
@@ -47,24 +49,21 @@ class MySQLService:
         search_for_user = f"SELECT user FROM mysql.user WHERE user='{username}'"
         search_for_db = f"SHOW DATABASES LIKE '{username}'"
         # CREATE USER can't be used in a query with multiple statements
-        create_user_commands = [
-            f"CREATE USER '{username}'@'localhost' IDENTIFIED BY %(password)s",
-            f"CREATE USER '{username}'@'%' IDENTIFIED BY %(password)s",
-        ]
+        create_local_user_cmd = f"CREATE USER '{username}'@'localhost' IDENTIFIED VIA unix_socket"
+        create_user_cmd = f"CREATE USER '{username}'@'%' IDENTIFIED BY %(password)s"
         create_database = f"""
         CREATE DATABASE {username};
-        GRANT ALL PRIVILEGES ON {username}.* TO '{username}'@'localhost';
+        GRANT ALL PRIVILEGES ON {username}.* TO '{username}'@'localhost' IDENTIFIED VIA unix_socket;
         GRANT ALL PRIVILEGES ON {username}.* TO '{username}'@'%';
         """
 
         with self.mysql_connection() as con, con.cursor() as cursor:
-            if response_is_empty(search_for_user, con):
-                for cmd in create_user_commands:
-                    cursor.execute(cmd, {'password': password})
-                if response_is_empty(search_for_db, con):
-                    cursor.execute(create_database)
-            else:
+            if not response_is_empty(search_for_user, con):
                 raise UserAlreadyExistsError()
+            cursor.execute(create_local_user_cmd)
+            cursor.execute(create_user_cmd, {'password': password})
+            if response_is_empty(search_for_db, con):
+                cursor.execute(create_database)
             return password
 
     def reset_db_passwd(self, username: str) -> str:
@@ -76,10 +75,9 @@ class MySQLService:
         """
 
         with self.mysql_connection() as con, con.cursor() as cursor:
-            if not response_is_empty(search_for_user, con):
-                cursor.execute(reset_password, {'password': password})
-            else:
+            if response_is_empty(search_for_user, con):
                 raise UserNotFoundError(username)
+            cursor.execute(reset_password, {'password': password})
             return password
 
     def delete_db(self, username: str):

ceod/db/PostgreSQLService.py

@@ -39,9 +39,11 @@ class PostgreSQLService:
             )
             con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
             yield con
+        # unable to connect / invalid credentials / user does not exist
         except OperationalError as e:
             logger.error(e)
             raise DatabaseConnectionError()
+        # invalid permissions for action
         except ProgrammingError as e:
             logger.error(e)
             raise DatabasePermissionError()

ceod/model/User.py

@@ -56,7 +56,6 @@ class User:
 
         self.ldap_srv = component.getUtility(ILDAPService)
         self.krb_srv = component.getUtility(IKerberosService)
-        self.file_srv = component.getUtility(IFileService)
 
     def to_dict(self, get_forwarding_addresses: bool = False) -> Dict:
         data = {
@@ -103,10 +102,12 @@ class User:
         self.krb_srv.change_password(self.uid, password)
 
     def create_home_dir(self):
-        self.file_srv.create_home_dir(self)
+        file_srv = component.getUtility(IFileService)
+        file_srv.create_home_dir(self)
 
     def delete_home_dir(self):
-        self.file_srv.delete_home_dir(self)
+        file_srv = component.getUtility(IFileService)
+        file_srv.delete_home_dir(self)
 
     def subscribe_to_mailing_list(self, mailing_list: str):
         component.getUtility(IMailmanService).subscribe(self.uid, mailing_list)
@@ -163,7 +164,9 @@ class User:
         self.positions = positions
 
     def get_forwarding_addresses(self) -> List[str]:
-        return self.file_srv.get_forwarding_addresses(self)
+        file_srv = component.getUtility(IFileService)
+        return file_srv.get_forwarding_addresses(self)
 
     def set_forwarding_addresses(self, addresses: List[str]):
-        self.file_srv.set_forwarding_addresses(self, addresses)
+        file_srv = component.getUtility(IFileService)
+        file_srv.set_forwarding_addresses(self, addresses)

tests/ceo/cli/test_db_mysql.py

@@ -0,0 +1,82 @@
+import os
+
+from click.testing import CliRunner
+from mysql.connector import connect
+from mysql.connector.errors import ProgrammingError
+import pytest
+
+from ceo.cli import cli
+
+
+def mysql_attempt_connection(host, username, password):
+    with connect(
+            host=host,
+            user=username,
+            password=password,
+    ) as con, con.cursor() as cur:
+        cur.execute("SHOW DATABASES")
+        response = cur.fetchall()
+        assert len(response) == 2
+
+        with pytest.raises(ProgrammingError):
+            cur.execute("CREATE DATABASE new_db")
+
+
+def test_mysql(cli_setup, cfg, ldap_user):
+    runner = CliRunner()
+
+    username = ldap_user.uid
+    os.makedirs(ldap_user.home_directory)
+    host = cfg.get("mysql_host")
+    info_file_path = os.path.join(ldap_user.home_directory, "ceo-mysql-info")
+    assert not os.path.isfile(info_file_path)
+
+    # create database for user
+    result = runner.invoke(cli, ['mysql', 'create', username], input='y\n')
+    assert result.exit_code == 0
+    assert os.path.isfile(info_file_path)
+
+    response_arr = result.output.split()
+    passwd = response_arr[response_arr.index("Password:") + 1]
+    with open(info_file_path, 'r') as file:
+        old_info = file.read()
+
+    expected = f"""Are you sure you want to create a MySQL database for {username}? [y/N]: y
+MySQL database created.
+Connection Information:
+
+Database: {username}
+Username: {username}
+Password: {passwd}
+Host: {host}
+
+These settings have been written to {info_file_path}.
+"""
+
+    assert result.output == expected
+    mysql_attempt_connection(host, username, passwd)
+
+    # perform password reset for user
+    # confirm once to reset password, another to overwrite the file
+    result = runner.invoke(cli, ['mysql', 'pwreset', username], input="y\ny\n")
+    assert result.exit_code == 0
+
+    response_arr = result.output.split()
+    new_passwd = response_arr[response_arr.index("Password:") + 1]
+    with open(info_file_path, 'r') as file:
+        new_info = file.read()
+
+    assert new_passwd != passwd
+    assert old_info != new_info
+    mysql_attempt_connection(host, username, new_passwd)
+
+    # delete database and file
+    result = runner.invoke(cli, ['mysql', 'delete', username], input="y\n")
+    assert result.exit_code == 0
+
+    # user should be deleted
+    with pytest.raises(ProgrammingError):
+        mysql_attempt_connection(host, username, passwd)
+
+    os.remove(info_file_path)
+    os.rmdir(ldap_user.home_directory)

tests/ceo/cli/test_db_postgresql.py

@@ -0,0 +1,84 @@
+import pytest
+import os
+
+from click.testing import CliRunner
+from ceo.cli import cli
+
+from psycopg2 import connect, OperationalError, ProgrammingError
+
+
+def psql_attempt_connection(host, username, password):
+    con = connect(
+        host=host,
+        user=username,
+        password=password,
+    )
+    con.autocommit = True
+    with con.cursor() as cur:
+        cur.execute("SELECT datname FROM pg_database")
+        response = cur.fetchall()
+        # 3 of the 4 are postgres, template0, template1
+        assert len(response) == 4
+        with pytest.raises(ProgrammingError):
+            cur.execute("CREATE DATABASE new_db")
+    con.close()
+
+
+def test_postgresql(cli_setup, cfg, ldap_user):
+    runner = CliRunner()
+
+    username = ldap_user.uid
+    os.makedirs(ldap_user.home_directory)
+    host = cfg.get("postgresql_host")
+    info_file_path = os.path.join(ldap_user.home_directory, "ceo-postgresql-info")
+    assert not os.path.isfile(info_file_path)
+
+    # create database for user
+    result = runner.invoke(cli, ['postgresql', 'create', username], input='y\n')
+    assert result.exit_code == 0
+    assert os.path.isfile(info_file_path)
+
+    response_arr = result.output.split()
+    passwd = response_arr[response_arr.index("Password:") + 1]
+    with open(info_file_path, 'r') as file:
+        old_info = file.read()
+
+    expected = f"""Are you sure you want to create a PostgreSQL database for {username}? [y/N]: y
+PostgreSQL database created.
+Connection Information:
+
+Database: {username}
+Username: {username}
+Password: {passwd}
+Host: {host}
+
+These settings have been written to {info_file_path}.
+"""
+
+    assert result.output == expected
+    psql_attempt_connection(host, username, passwd)
+
+    # perform password reset for user
+    # confirm once to reset password, another to overwrite the file
+    result = runner.invoke(cli, ['postgresql', 'pwreset', username], input="y\ny\n")
+    assert result.exit_code == 0
+
+    response_arr = result.output.split()
+    new_passwd = response_arr[response_arr.index("Password:") + 1]
+    with open(info_file_path, 'r') as file:
+        new_info = file.read()
+
+    assert new_passwd != passwd
+    assert old_info != new_info
+    psql_attempt_connection(host, username, new_passwd)
+
+    # delete database and file
+    result = runner.invoke(cli, ['postgresql', 'delete', username], input="y\n")
+    assert result.exit_code == 0
+
+    # user should be deleted
+    with pytest.raises(OperationalError):
+        psql_attempt_connection(host, username, passwd)
+
+    os.remove(info_file_path)
+    os.rmdir(ldap_user.home_directory)

tests/ceo_dev.ini

@@ -5,6 +5,7 @@ uw_domain = uwaterloo.internal
 [ceod]
 # this is the host with the ceod/admin Kerberos key
 admin_host = phosphoric-acid
+database_host = coffee
 use_https = false
 port = 9987
 
@@ -12,3 +13,9 @@ port = 9987
 required = president,vice-president,sysadmin
 available = president,vice-president,treasurer,secretary,
             sysadmin,cro,librarian,imapd,webmaster,offsck
+
+[mysql]
+host = coffee
+
+[postgresql]
+host = coffee

tests/ceod_dev.ini

@@ -13,7 +13,7 @@ port = 9987
 
 [ldap]
 admin_principal = ceod/admin
-server_url = ldap://ldap-master.csclub.internal
+server_url = ldap://auth1.csclub.internal
 sasl_realm = CSCLUB.INTERNAL
 users_base = ou=People,dc=csclub,dc=internal
 groups_base = ou=Group,dc=csclub,dc=internal