move all tests to top-level folder

ceod/api/app_factory.py

@@ -18,67 +18,86 @@ def create_app(flask_config={}):
     app = Flask(__name__)
     app.config.from_mapping(flask_config)
 
+    if not app.config.get('TESTING'):
+        register_services(app)
+
+    cfg = component.getUtility(IConfig)
+    fqdn = socket.getfqdn()
+    os.environ['KRB5_KTNAME'] = '/etc/krb5.keytab'
+    init_kerberos(app, service='ceod', hostname=fqdn)
+
+    hostname = socket.gethostname()
+    # Only ceod_admin_host should serve the /api/members endpoints because
+    # it needs to run kadmin
+    if hostname == cfg.get('ceod_admin_host'):
+        from ceod.api import members
+        app.register_blueprint(members.bp, url_prefix='/api/members')
+
+    if hostname == cfg.get('ceod_mailman_host'):
+        # Only offer mailman API if this host is running Mailman
+        from ceod.api import mailman
+        app.register_blueprint(mailman.bp, url_prefix='/api/mailman')
+
+    from ceod.api import uwldap
+    app.register_blueprint(uwldap.bp, url_prefix='/api/uwldap')
+
+    register_error_handlers(app)
+
+    @app.route('/ping')
+    def ping():
+        """Health check"""
+        return 'pong\n'
+
+    return app
+
+
+def register_services(app):
+    # Config
     if app.config.get('ENV') == 'development' and 'CEOD_CONFIG' not in os.environ:
-        with importlib.resources.path('tests_common', 'ceod_dev.ini') as p:
+        with importlib.resources.path('tests', 'ceod_dev.ini') as p:
             config_file = p.__fspath__()
     else:
         config_file = None
     cfg = Config(config_file)
     component.provideUtility(cfg, IConfig)
 
-    init_kerberos(app, service='ceod')
-
+    # KerberosService
+    if 'KRB5_KTNAME' not in os.environ:
+        os.environ['KRB5_KTNAME'] = '/etc/krb5.keytab'
     hostname = socket.gethostname()
+    fqdn = socket.getfqdn()
     # Only ceod_admin_host has the ceod/admin key in its keytab
     if hostname == cfg.get('ceod_admin_host'):
-        krb_srv = KerberosService(cfg.get('ldap_admin_principal'))
-
-        from ceod.api import members
-        app.register_blueprint(members.bp, url_prefix='/api/members')
+        principal = cfg.get('ldap_admin_principal')
     else:
-        fqdn = socket.getfqdn()
-        krb_srv = KerberosService(f'ceod/{fqdn}')
+        principal = f'ceod/{fqdn}'
+    krb_srv = KerberosService(principal)
     component.provideUtility(krb_srv, IKerberosService)
 
-    # Any host can use LDAPService, but only ceod_admin_host can write
+    # LDAPService
     ldap_srv = LDAPService()
     component.provideUtility(ldap_srv, ILDAPService)
 
+    # HTTPService
     http_client = HTTPClient()
     component.provideUtility(http_client, IHTTPClient)
 
-    # Only instantiate FileService if this host has NFS no_root_squash
-    # If admin_host and fs_root_host become separate, we will need
-    # to create a RemoteFileService
+    # FileService
     if hostname == cfg.get('ceod_fs_root_host'):
         file_srv = FileService()
         component.provideUtility(file_srv, IFileService)
 
-    # Only offer mailman API if this host is running Mailman
+    # MailmanService
     if hostname == cfg.get('ceod_mailman_host'):
         mailman_srv = MailmanService()
-        component.provideUtility(mailman_srv, IMailmanService)
-
-        from ceod.api import mailman
-        app.register_blueprint(mailman.bp, url_prefix='/api/mailman')
     else:
         mailman_srv = RemoteMailmanService()
-        component.provideUtility(mailman_srv, IMailmanService)
+    component.provideUtility(mailman_srv, IMailmanService)
 
+    # MailService
     mail_srv = MailService()
     component.provideUtility(mail_srv, IMailService)
 
+    # UWLDAPService
     uwldap_srv = UWLDAPService()
     component.provideUtility(uwldap_srv, IUWLDAPService)
-
-    from ceod.api import uwldap
-    app.register_blueprint(uwldap.bp, url_prefix='/api/uwldap')
-
-    register_error_handlers(app)
-
-    @app.route('/ping')
-    def ping():
-        """Health check"""
-        return 'pong\n'
-
-    return app

ceod/model/KerberosService.py

@@ -11,11 +11,11 @@ class KerberosService:
     def __init__(
         self,
         admin_principal: str,
-        cache_file: str = '/run/ceod/krb5_cache',
+        cache_dir: str = '/run/ceod/krb5_cache',
     ):
         self.admin_principal = admin_principal
-        os.makedirs(os.path.dirname(cache_file), exist_ok=True)
-        os.putenv('KRB5CCNAME', 'FILE:' + cache_file)
+        os.makedirs(cache_dir, exist_ok=True)
+        os.environ['KRB5CCNAME'] = 'DIR:' + cache_dir
         self.kinit()
 
     def kinit(self):
@@ -27,6 +27,7 @@ class KerberosService:
             '-pw', password,
             '-policy', 'default',
             '+needchange',
+            '+requires_preauth',
             principal
         ], check=True)
 

ceod/model/test/conftest.py

@@ -1 +0,0 @@
-from tests_common.fixtures import *

ceod/transactions/members/AddMemberTransaction.py

@@ -44,7 +44,7 @@ class AddMemberTransaction(AbstractTransaction):
         self.forwarding_addresses = forwarding_addresses
         self.member = None
         self.group = None
-        self.new_member_list = cfg.get('new_member_list')
+        self.new_member_list = cfg.get('mailman3_new_member_list')
         self.mail_srv = component.getUtility(IMailService)
 
     def child_execute_iter(self):

clear_cache.sh

@@ -1,2 +1,3 @@
 #!/bin/sh
 find ceo* -type d -name __pycache__ -execdir rm -r '{}' \;
+rm -rf .pytest_cache

dev-requirements.txt

@@ -2,3 +2,5 @@ flake8==3.9.2
 setuptools==40.8.0
 wheel==0.36.2
 pytest==6.2.4
+aiosmtpd==1.4.2
+aiohttp==3.7.4.post0

tests/MockMailmanServer.py

@@ -0,0 +1,49 @@
+import asyncio
+from threading import Thread
+from aiohttp import web
+
+
+class MockMailmanServer:
+    def __init__(self):
+        self.app = web.Application()
+        self.app.add_routes([
+            web.post('/members', self.subscribe),
+            web.delete('/lists/{mailing_list}/member/{address}', self.unsubscribe),
+        ])
+        self.runner = web.AppRunner(self.app)
+        self.loop = asyncio.new_event_loop()
+
+        self.subscriptions = []
+
+    def _start_loop(self):
+        asyncio.set_event_loop(self.loop)
+        self.loop.run_until_complete(self.runner.setup())
+        site = web.TCPSite(self.runner, 'localhost', 8002)
+        self.loop.run_until_complete(site.start())
+        self.loop.run_forever()
+
+    def start(self):
+        t = Thread(target=self._start_loop)
+        t.start()
+
+    def stop(self):
+        self.loop.call_soon_threadsafe(self.loop.stop)
+
+    async def subscribe(self, request):
+        body = await request.post()
+        subscriber = body['subscriber']
+        if subscriber in self.subscriptions:
+            return web.json_response({
+                'description': 'user is already subscribed',
+            }, status=409)
+        self.subscriptions.append(subscriber)
+        return web.json_response({'status': 'OK'})
+
+    async def unsubscribe(self, request):
+        subscriber = request.match_info['address']
+        if subscriber not in self.subscriptions:
+            return web.json_response({
+                'description': 'user is not subscribed',
+            }, status=404)
+        self.subscriptions.remove(subscriber)
+        return web.json_response({'status': 'OK'})

tests/MockSMTPServer.py

@@ -0,0 +1,27 @@
+from aiosmtpd.controller import Controller
+
+
+class MockSMTPServer:
+    def __init__(self, hostname='localhost', port=8025):
+        self.messages = []
+        self.controller = Controller(MockHandler(self), hostname, port)
+
+    def start(self):
+        self.controller.start()
+
+    def stop(self):
+        self.controller.stop()
+
+
+class MockHandler:
+    def __init__(self, mock_server):
+        self.mock_server = mock_server
+
+    async def handle_DATA(self, server, session, envelope):
+        msg = {
+            'from': envelope.mail_from,
+            'to': envelope.rcpt_tos[0],
+            'content': envelope.content.decode(),
+        }
+        self.mock_server.messages.append(msg)
+        return '250 Message accepted for delivery'

tests/__init__.py

@@ -0,0 +1,2 @@
+from .MockSMTPServer import MockSMTPServer
+from .MockMailmanServer import MockMailmanServer

tests/ceod/api/test_members.py

@@ -0,0 +1,23 @@
+import pytest
+
+
+def test_members_get_user(client):
+    status, data = client.get('/api/members/no_such_user')
+    assert status == 404
+    assert data['error'] == 'user not found'
+
+
+@pytest.fixture(scope='session')
+def create_user_resp(client):
+    return client.post('/api/members', json={
+        'uid': 'test_jdoe',
+        'cn': 'John Doe',
+        'program': 'Math',
+        'terms': ['s2021'],
+    })
+
+
+# def test_create_user(create_user_resp):
+#     status, data = create_user_resp
+#     assert status == 200
+#     # TODO: check response contents

tests/ceod/model/test_mail.py

@@ -0,0 +1,10 @@
+def test_welcome_message(cfg, mock_mail_server, mail_srv, simple_user):
+    base_domain = cfg.get('base_domain')
+    mail_srv.send_welcome_message_to(simple_user)
+    msg = mock_mail_server.messages[0]
+    assert msg['from'] == f'exec@{base_domain}'
+    assert msg['to'] == f'{simple_user.uid}@{base_domain}'
+    # make sure that templating was applied correctly
+    first_name = simple_user.cn.split()[0]
+    assert f'Hello {first_name}' in msg['content']
+    mock_mail_server.messages.clear()

tests/ceod/model/test_mailman.py

@@ -3,7 +3,7 @@ import pytest
 from ceo_common.errors import UserAlreadySubscribedError, UserNotSubscribedError
 
 
-def test_user_mailing_lists(ldap_user):
+def test_user_mailing_lists(mailman_srv, ldap_user):
     user = ldap_user
 
     user.subscribe_to_mailing_list('csc-general')

tests/ceod_test_local.ini

@@ -37,7 +37,7 @@ smtp_url = smtp://localhost:8025
 smtp_starttls = false
 
 [mailman3]
-api_base_url = http://localhost:8001/3.1
+api_base_url = http://localhost:8002
 api_username = restadmin
 api_password = mailman3
 new_member_list = csc-general

tests/conftest.py

@@ -8,23 +8,27 @@ import socket
 from zope import component
 
 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
-    IFileService, IMailmanService, IHTTPClient, IUWLDAPService
+    IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService
 from ceo_common.model import Config, RemoteMailmanService, HTTPClient
+from ceod.api import create_app
 from ceod.model import KerberosService, LDAPService, FileService, User, \
-    MailmanService, Group, UWLDAPService, UWLDAPRecord
+    MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService
 from ceod.model.utils import strings_to_bytes
+from .MockSMTPServer import MockSMTPServer
+from .MockMailmanServer import MockMailmanServer
+from .conftest_ceod_api import *
 
 
-@pytest.fixture(autouse=True, scope='session')
+@pytest.fixture(scope='session')
 def cfg():
-    with importlib.resources.path('tests_common', 'ceod_test_local.ini') as p:
+    with importlib.resources.path('tests', 'ceod_test_local.ini') as p:
         config_file = p.__fspath__()
     _cfg = Config(config_file)
     component.provideUtility(_cfg, IConfig)
     return _cfg
 
 
-@pytest.fixture(autouse=True, scope='session')
+@pytest.fixture(scope='session')
 def krb_srv(cfg):
     # we need to be root to read the keytab
     assert os.geteuid() == 0
@@ -33,13 +37,12 @@ def krb_srv(cfg):
         principal = 'ceod/admin'
     else:
         principal = 'ceod/' + socket.getfqdn()
-    cache_file = '/tmp/ceod_test/krb5_cache'
-    if os.path.isfile(cache_file):
-        os.unlink(cache_file)
-    krb = KerberosService(principal, cache_file)
+    cache_dir = '/tmp/ceod_test/krb5_cache'
+    shutil.rmtree(cache_dir, ignore_errors=True)
+    krb = KerberosService(principal, cache_dir)
     component.provideUtility(krb, IKerberosService)
     yield krb
-    os.unlink(cache_file)
+    shutil.rmtree(cache_dir)
 
 
 def recursively_delete_subtree(conn: ldap.ldapobject.LDAPObject, base_dn: str):
@@ -52,7 +55,7 @@ def recursively_delete_subtree(conn: ldap.ldapobject.LDAPObject, base_dn: str):
         pass
 
 
-@pytest.fixture(autouse=True, scope='session')
+@pytest.fixture(scope='session')
 def ldap_srv(cfg, krb_srv):
     conn = ldap.initialize(cfg.get('ldap_server_url'))
     conn.sasl_gssapi_bind_s()
@@ -76,7 +79,7 @@ def ldap_srv(cfg, krb_srv):
     recursively_delete_subtree(conn, groups_base)
 
 
-@pytest.fixture(autouse=True, scope='session')
+@pytest.fixture(scope='session')
 def file_srv(cfg):
     _file_srv = FileService()
     component.provideUtility(_file_srv, IFileService)
@@ -90,6 +93,82 @@ def file_srv(cfg):
     shutil.rmtree(clubs_home, ignore_errors=True)
 
 
+@pytest.fixture(scope='session')
+def http_client(cfg):
+    client = HTTPClient()
+    component.provideUtility(client, IHTTPClient)
+    return
+
+
+@pytest.fixture(scope='session')
+def mock_mailman_server():
+    server = MockMailmanServer()
+    server.start()
+    yield server
+    server.stop()
+
+
+@pytest.fixture(scope='session')
+def mailman_srv(mock_mailman_server, cfg, http_client):
+    # TODO: test the RemoteMailmanService as well
+    mailman = MailmanService()
+    component.provideUtility(mailman, IMailmanService)
+    return mailman
+
+
+@pytest.fixture(scope='session')
+def uwldap_srv(cfg, ldap_srv):
+    conn = ldap.initialize(cfg.get('uwldap_server_url'))
+    conn.sasl_gssapi_bind_s()
+    base_dn = cfg.get('uwldap_base')
+    ou = base_dn.split(',', 1)[0].split('=')[1]
+
+    recursively_delete_subtree(conn, base_dn)
+
+    conn.add_s(base_dn, ldap.modlist.addModlist({
+        'objectClass': [b'organizationalUnit'],
+        'ou': [ou.encode()]
+    }))
+    _uwldap_srv = UWLDAPService()
+    component.provideUtility(_uwldap_srv, IUWLDAPService)
+    yield _uwldap_srv
+
+    recursively_delete_subtree(conn, base_dn)
+
+
+@pytest.fixture(scope='session')
+def mock_mail_server():
+    mock_server = MockSMTPServer()
+    mock_server.start()
+    yield mock_server
+    mock_server.stop()
+
+
+@pytest.fixture(scope='session')
+def mail_srv(cfg, mock_mail_server):
+    _mail_srv = MailService()
+    component.provideUtility(_mail_srv, IMailService)
+    return _mail_srv
+
+
+@pytest.fixture(autouse=True, scope='session')
+def app(
+    cfg,
+    krb_srv,
+    ldap_srv,
+    file_srv,
+    mailman_srv,
+    uwldap_srv,
+    mail_srv,
+):
+    # need to be root to read keytab
+    assert os.geteuid() == 0
+    app = create_app({
+        'TESTING': True,
+    })
+    return app
+
+
 @pytest.fixture
 def simple_user():
     return User(
@@ -123,24 +202,6 @@ def krb_user(simple_user):
     simple_user.remove_from_kerberos()
 
 
-@pytest.fixture(scope='session')
-def http_client():
-    client = HTTPClient()
-    component.provideUtility(client, IHTTPClient)
-    return
-
-
-@pytest.fixture(autouse=True, scope='session')
-def mailman_srv(cfg, http_client):
-    if socket.gethostname() == cfg.get('ceod_mailman_host'):
-        # TODO: use a mock server on drone.io
-        mailman = MailmanService()
-    else:
-        mailman = RemoteMailmanService()
-    component.provideUtility(mailman, IMailmanService)
-    return mailman
-
-
 @pytest.fixture
 def simple_group():
     return Group(
@@ -156,26 +217,6 @@ def ldap_group(simple_group):
     simple_group.remove_from_ldap()
 
 
-@pytest.fixture(scope='session')
-def uwldap_srv(cfg, ldap_srv):
-    conn = ldap.initialize(cfg.get('uwldap_server_url'))
-    conn.sasl_gssapi_bind_s()
-    base_dn = cfg.get('uwldap_base')
-    ou = base_dn.split(',', 1)[0].split('=')[1]
-
-    recursively_delete_subtree(conn, base_dn)
-
-    conn.add_s(base_dn, ldap.modlist.addModlist({
-        'objectClass': [b'organizationalUnit'],
-        'ou': [ou.encode()]
-    }))
-    _uwldap_srv = UWLDAPService()
-    component.provideUtility(_uwldap_srv, IUWLDAPService)
-    yield _uwldap_srv
-
-    recursively_delete_subtree(conn, base_dn)
-
-
 @pytest.fixture
 def uwldap_user(cfg, uwldap_srv):
     conn = ldap.initialize(cfg.get('uwldap_server_url'))

tests/conftest_ceod_api.py

@@ -0,0 +1,66 @@
+import json
+import socket
+
+from flask.testing import FlaskClient
+import gssapi
+import pytest
+from requests import Request
+from requests_gssapi import HTTPSPNEGOAuth
+from zope import component
+
+from ceo_common.interfaces import IConfig
+
+
+@pytest.fixture(scope='session')
+def client(app):
+    app_client = app.test_client()
+    return CeodTestClient(app_client)
+
+
+class CeodTestClient:
+    def __init__(self, app_client: FlaskClient):
+        cfg = component.getUtility(IConfig)
+        self.client = app_client
+        self.admin_principal = cfg.get('ldap_admin_principal')
+        # this is only used for the HTTPSNEGOAuth
+        self.base_url = f'http://{socket.getfqdn()}'
+        self.cached_auth = {}
+
+    def get_auth(self, principal):
+        if principal in self.cached_auth:
+            return self.cached_auth[principal]
+        name = gssapi.Name(principal)
+        creds = gssapi.Credentials(name=name, usage='initiate')
+        auth = HTTPSPNEGOAuth(
+            opportunistic_auth=True,
+            target_name='ceod',
+            creds=creds,
+        )
+        self.cached_auth[principal] = auth
+        return auth
+
+    def get_headers(self, principal):
+        # method doesn't matter here because we just need the headers
+        req = Request('GET', self.base_url, auth=self.get_auth(principal))
+        return req.prepare().headers.items()
+
+    def request(self, method, path, principal, **kwargs):
+        if principal is None:
+            principal = self.admin_principal
+        resp = self.client.open(
+            path, method=method, headers=self.get_headers(principal), **kwargs)
+        status = int(resp.status.split(' ', 1)[0])
+        try:
+            data = json.loads(resp.data)
+        except json.JSONDecodeError:
+            data = resp.data.decode()
+        return status, data
+
+    def get(self, path, principal=None, **kwargs):
+        return self.request('GET', path, principal, **kwargs)
+
+    def post(self, path, principal=None, **kwargs):
+        return self.request('POST', path, principal, **kwargs)
+
+    def delete(self, path, principal=None, **kwargs):
+        return self.request('DELETE', path, principal, **kwargs)