From 6cdb41d47b04de024d3f38722054819c1f623a4b Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Sat, 14 Aug 2021 00:11:56 +0000 Subject: [PATCH] move all tests to top-level folder --- ceod/api/app_factory.py | 103 +++++++------ ceod/model/KerberosService.py | 7 +- ceod/model/test/conftest.py | 1 - .../members/AddMemberTransaction.py | 2 +- clear_cache.sh | 1 + dev-requirements.txt | 2 + tests/MockMailmanServer.py | 49 ++++++ tests/MockSMTPServer.py | 27 ++++ tests/__init__.py | 2 + .../model/test => tests/ceod/api}/__init__.py | 0 tests/ceod/api/test_members.py | 23 +++ .../ceod/model}/__init__.py | 0 .../test => tests/ceod/model}/test_group.py | 0 tests/ceod/model/test_mail.py | 10 ++ .../test => tests/ceod/model}/test_mailman.py | 2 +- .../test => tests/ceod/model}/test_user.py | 0 .../test => tests/ceod/model}/test_uwldap.py | 0 {tests_common => tests}/ceod_dev.ini | 0 {tests_common => tests}/ceod_test_local.ini | 2 +- tests_common/fixtures.py => tests/conftest.py | 141 +++++++++++------- tests/conftest_ceod_api.py | 66 ++++++++ 21 files changed, 339 insertions(+), 99 deletions(-) delete mode 100644 ceod/model/test/conftest.py create mode 100644 tests/MockMailmanServer.py create mode 100644 tests/MockSMTPServer.py create mode 100644 tests/__init__.py rename {ceod/model/test => tests/ceod/api}/__init__.py (100%) create mode 100644 tests/ceod/api/test_members.py rename {tests_common => tests/ceod/model}/__init__.py (100%) rename {ceod/model/test => tests/ceod/model}/test_group.py (100%) create mode 100644 tests/ceod/model/test_mail.py rename {ceod/model/test => tests/ceod/model}/test_mailman.py (89%) rename {ceod/model/test => tests/ceod/model}/test_user.py (100%) rename {ceod/model/test => tests/ceod/model}/test_uwldap.py (100%) rename {tests_common => tests}/ceod_dev.ini (100%) rename {tests_common => tests}/ceod_test_local.ini (95%) rename tests_common/fixtures.py => tests/conftest.py (75%) create mode 100644 tests/conftest_ceod_api.py diff --git a/ceod/api/app_factory.py b/ceod/api/app_factory.py index 37726f3..3189ea6 100644 --- a/ceod/api/app_factory.py +++ b/ceod/api/app_factory.py @@ -18,58 +18,25 @@ def create_app(flask_config={}): app = Flask(__name__) app.config.from_mapping(flask_config) - if app.config.get('ENV') == 'development' and 'CEOD_CONFIG' not in os.environ: - with importlib.resources.path('tests_common', 'ceod_dev.ini') as p: - config_file = p.__fspath__() - else: - config_file = None - cfg = Config(config_file) - component.provideUtility(cfg, IConfig) + if not app.config.get('TESTING'): + register_services(app) - init_kerberos(app, service='ceod') + cfg = component.getUtility(IConfig) + fqdn = socket.getfqdn() + os.environ['KRB5_KTNAME'] = '/etc/krb5.keytab' + init_kerberos(app, service='ceod', hostname=fqdn) hostname = socket.gethostname() - # Only ceod_admin_host has the ceod/admin key in its keytab + # Only ceod_admin_host should serve the /api/members endpoints because + # it needs to run kadmin if hostname == cfg.get('ceod_admin_host'): - krb_srv = KerberosService(cfg.get('ldap_admin_principal')) - from ceod.api import members app.register_blueprint(members.bp, url_prefix='/api/members') - else: - fqdn = socket.getfqdn() - krb_srv = KerberosService(f'ceod/{fqdn}') - component.provideUtility(krb_srv, IKerberosService) - # Any host can use LDAPService, but only ceod_admin_host can write - ldap_srv = LDAPService() - component.provideUtility(ldap_srv, ILDAPService) - - http_client = HTTPClient() - component.provideUtility(http_client, IHTTPClient) - - # Only instantiate FileService if this host has NFS no_root_squash - # If admin_host and fs_root_host become separate, we will need - # to create a RemoteFileService - if hostname == cfg.get('ceod_fs_root_host'): - file_srv = FileService() - component.provideUtility(file_srv, IFileService) - - # Only offer mailman API if this host is running Mailman if hostname == cfg.get('ceod_mailman_host'): - mailman_srv = MailmanService() - component.provideUtility(mailman_srv, IMailmanService) - + # Only offer mailman API if this host is running Mailman from ceod.api import mailman app.register_blueprint(mailman.bp, url_prefix='/api/mailman') - else: - mailman_srv = RemoteMailmanService() - component.provideUtility(mailman_srv, IMailmanService) - - mail_srv = MailService() - component.provideUtility(mail_srv, IMailService) - - uwldap_srv = UWLDAPService() - component.provideUtility(uwldap_srv, IUWLDAPService) from ceod.api import uwldap app.register_blueprint(uwldap.bp, url_prefix='/api/uwldap') @@ -82,3 +49,55 @@ def create_app(flask_config={}): return 'pong\n' return app + + +def register_services(app): + # Config + if app.config.get('ENV') == 'development' and 'CEOD_CONFIG' not in os.environ: + with importlib.resources.path('tests', 'ceod_dev.ini') as p: + config_file = p.__fspath__() + else: + config_file = None + cfg = Config(config_file) + component.provideUtility(cfg, IConfig) + + # KerberosService + if 'KRB5_KTNAME' not in os.environ: + os.environ['KRB5_KTNAME'] = '/etc/krb5.keytab' + hostname = socket.gethostname() + fqdn = socket.getfqdn() + # Only ceod_admin_host has the ceod/admin key in its keytab + if hostname == cfg.get('ceod_admin_host'): + principal = cfg.get('ldap_admin_principal') + else: + principal = f'ceod/{fqdn}' + krb_srv = KerberosService(principal) + component.provideUtility(krb_srv, IKerberosService) + + # LDAPService + ldap_srv = LDAPService() + component.provideUtility(ldap_srv, ILDAPService) + + # HTTPService + http_client = HTTPClient() + component.provideUtility(http_client, IHTTPClient) + + # FileService + if hostname == cfg.get('ceod_fs_root_host'): + file_srv = FileService() + component.provideUtility(file_srv, IFileService) + + # MailmanService + if hostname == cfg.get('ceod_mailman_host'): + mailman_srv = MailmanService() + else: + mailman_srv = RemoteMailmanService() + component.provideUtility(mailman_srv, IMailmanService) + + # MailService + mail_srv = MailService() + component.provideUtility(mail_srv, IMailService) + + # UWLDAPService + uwldap_srv = UWLDAPService() + component.provideUtility(uwldap_srv, IUWLDAPService) diff --git a/ceod/model/KerberosService.py b/ceod/model/KerberosService.py index f038804..af16274 100644 --- a/ceod/model/KerberosService.py +++ b/ceod/model/KerberosService.py @@ -11,11 +11,11 @@ class KerberosService: def __init__( self, admin_principal: str, - cache_file: str = '/run/ceod/krb5_cache', + cache_dir: str = '/run/ceod/krb5_cache', ): self.admin_principal = admin_principal - os.makedirs(os.path.dirname(cache_file), exist_ok=True) - os.putenv('KRB5CCNAME', 'FILE:' + cache_file) + os.makedirs(cache_dir, exist_ok=True) + os.environ['KRB5CCNAME'] = 'DIR:' + cache_dir self.kinit() def kinit(self): @@ -27,6 +27,7 @@ class KerberosService: '-pw', password, '-policy', 'default', '+needchange', + '+requires_preauth', principal ], check=True) diff --git a/ceod/model/test/conftest.py b/ceod/model/test/conftest.py deleted file mode 100644 index 592da9c..0000000 --- a/ceod/model/test/conftest.py +++ /dev/null @@ -1 +0,0 @@ -from tests_common.fixtures import * diff --git a/ceod/transactions/members/AddMemberTransaction.py b/ceod/transactions/members/AddMemberTransaction.py index 758851b..c522d7e 100644 --- a/ceod/transactions/members/AddMemberTransaction.py +++ b/ceod/transactions/members/AddMemberTransaction.py @@ -44,7 +44,7 @@ class AddMemberTransaction(AbstractTransaction): self.forwarding_addresses = forwarding_addresses self.member = None self.group = None - self.new_member_list = cfg.get('new_member_list') + self.new_member_list = cfg.get('mailman3_new_member_list') self.mail_srv = component.getUtility(IMailService) def child_execute_iter(self): diff --git a/clear_cache.sh b/clear_cache.sh index 3d1f495..3d70e9d 100755 --- a/clear_cache.sh +++ b/clear_cache.sh @@ -1,2 +1,3 @@ #!/bin/sh find ceo* -type d -name __pycache__ -execdir rm -r '{}' \; +rm -rf .pytest_cache diff --git a/dev-requirements.txt b/dev-requirements.txt index 3c7b184..8392fb6 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,3 +2,5 @@ flake8==3.9.2 setuptools==40.8.0 wheel==0.36.2 pytest==6.2.4 +aiosmtpd==1.4.2 +aiohttp==3.7.4.post0 diff --git a/tests/MockMailmanServer.py b/tests/MockMailmanServer.py new file mode 100644 index 0000000..7315afb --- /dev/null +++ b/tests/MockMailmanServer.py @@ -0,0 +1,49 @@ +import asyncio +from threading import Thread +from aiohttp import web + + +class MockMailmanServer: + def __init__(self): + self.app = web.Application() + self.app.add_routes([ + web.post('/members', self.subscribe), + web.delete('/lists/{mailing_list}/member/{address}', self.unsubscribe), + ]) + self.runner = web.AppRunner(self.app) + self.loop = asyncio.new_event_loop() + + self.subscriptions = [] + + def _start_loop(self): + asyncio.set_event_loop(self.loop) + self.loop.run_until_complete(self.runner.setup()) + site = web.TCPSite(self.runner, 'localhost', 8002) + self.loop.run_until_complete(site.start()) + self.loop.run_forever() + + def start(self): + t = Thread(target=self._start_loop) + t.start() + + def stop(self): + self.loop.call_soon_threadsafe(self.loop.stop) + + async def subscribe(self, request): + body = await request.post() + subscriber = body['subscriber'] + if subscriber in self.subscriptions: + return web.json_response({ + 'description': 'user is already subscribed', + }, status=409) + self.subscriptions.append(subscriber) + return web.json_response({'status': 'OK'}) + + async def unsubscribe(self, request): + subscriber = request.match_info['address'] + if subscriber not in self.subscriptions: + return web.json_response({ + 'description': 'user is not subscribed', + }, status=404) + self.subscriptions.remove(subscriber) + return web.json_response({'status': 'OK'}) diff --git a/tests/MockSMTPServer.py b/tests/MockSMTPServer.py new file mode 100644 index 0000000..4192660 --- /dev/null +++ b/tests/MockSMTPServer.py @@ -0,0 +1,27 @@ +from aiosmtpd.controller import Controller + + +class MockSMTPServer: + def __init__(self, hostname='localhost', port=8025): + self.messages = [] + self.controller = Controller(MockHandler(self), hostname, port) + + def start(self): + self.controller.start() + + def stop(self): + self.controller.stop() + + +class MockHandler: + def __init__(self, mock_server): + self.mock_server = mock_server + + async def handle_DATA(self, server, session, envelope): + msg = { + 'from': envelope.mail_from, + 'to': envelope.rcpt_tos[0], + 'content': envelope.content.decode(), + } + self.mock_server.messages.append(msg) + return '250 Message accepted for delivery' diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..6110bcd --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +from .MockSMTPServer import MockSMTPServer +from .MockMailmanServer import MockMailmanServer diff --git a/ceod/model/test/__init__.py b/tests/ceod/api/__init__.py similarity index 100% rename from ceod/model/test/__init__.py rename to tests/ceod/api/__init__.py diff --git a/tests/ceod/api/test_members.py b/tests/ceod/api/test_members.py new file mode 100644 index 0000000..d640c66 --- /dev/null +++ b/tests/ceod/api/test_members.py @@ -0,0 +1,23 @@ +import pytest + + +def test_members_get_user(client): + status, data = client.get('/api/members/no_such_user') + assert status == 404 + assert data['error'] == 'user not found' + + +@pytest.fixture(scope='session') +def create_user_resp(client): + return client.post('/api/members', json={ + 'uid': 'test_jdoe', + 'cn': 'John Doe', + 'program': 'Math', + 'terms': ['s2021'], + }) + + +# def test_create_user(create_user_resp): +# status, data = create_user_resp +# assert status == 200 +# # TODO: check response contents diff --git a/tests_common/__init__.py b/tests/ceod/model/__init__.py similarity index 100% rename from tests_common/__init__.py rename to tests/ceod/model/__init__.py diff --git a/ceod/model/test/test_group.py b/tests/ceod/model/test_group.py similarity index 100% rename from ceod/model/test/test_group.py rename to tests/ceod/model/test_group.py diff --git a/tests/ceod/model/test_mail.py b/tests/ceod/model/test_mail.py new file mode 100644 index 0000000..68aacdf --- /dev/null +++ b/tests/ceod/model/test_mail.py @@ -0,0 +1,10 @@ +def test_welcome_message(cfg, mock_mail_server, mail_srv, simple_user): + base_domain = cfg.get('base_domain') + mail_srv.send_welcome_message_to(simple_user) + msg = mock_mail_server.messages[0] + assert msg['from'] == f'exec@{base_domain}' + assert msg['to'] == f'{simple_user.uid}@{base_domain}' + # make sure that templating was applied correctly + first_name = simple_user.cn.split()[0] + assert f'Hello {first_name}' in msg['content'] + mock_mail_server.messages.clear() diff --git a/ceod/model/test/test_mailman.py b/tests/ceod/model/test_mailman.py similarity index 89% rename from ceod/model/test/test_mailman.py rename to tests/ceod/model/test_mailman.py index 0c30664..05517c0 100644 --- a/ceod/model/test/test_mailman.py +++ b/tests/ceod/model/test_mailman.py @@ -3,7 +3,7 @@ import pytest from ceo_common.errors import UserAlreadySubscribedError, UserNotSubscribedError -def test_user_mailing_lists(ldap_user): +def test_user_mailing_lists(mailman_srv, ldap_user): user = ldap_user user.subscribe_to_mailing_list('csc-general') diff --git a/ceod/model/test/test_user.py b/tests/ceod/model/test_user.py similarity index 100% rename from ceod/model/test/test_user.py rename to tests/ceod/model/test_user.py diff --git a/ceod/model/test/test_uwldap.py b/tests/ceod/model/test_uwldap.py similarity index 100% rename from ceod/model/test/test_uwldap.py rename to tests/ceod/model/test_uwldap.py diff --git a/tests_common/ceod_dev.ini b/tests/ceod_dev.ini similarity index 100% rename from tests_common/ceod_dev.ini rename to tests/ceod_dev.ini diff --git a/tests_common/ceod_test_local.ini b/tests/ceod_test_local.ini similarity index 95% rename from tests_common/ceod_test_local.ini rename to tests/ceod_test_local.ini index 07ead38..b82ba2d 100644 --- a/tests_common/ceod_test_local.ini +++ b/tests/ceod_test_local.ini @@ -37,7 +37,7 @@ smtp_url = smtp://localhost:8025 smtp_starttls = false [mailman3] -api_base_url = http://localhost:8001/3.1 +api_base_url = http://localhost:8002 api_username = restadmin api_password = mailman3 new_member_list = csc-general diff --git a/tests_common/fixtures.py b/tests/conftest.py similarity index 75% rename from tests_common/fixtures.py rename to tests/conftest.py index 02a25f3..8c7d27e 100644 --- a/tests_common/fixtures.py +++ b/tests/conftest.py @@ -8,23 +8,27 @@ import socket from zope import component from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \ - IFileService, IMailmanService, IHTTPClient, IUWLDAPService + IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService from ceo_common.model import Config, RemoteMailmanService, HTTPClient +from ceod.api import create_app from ceod.model import KerberosService, LDAPService, FileService, User, \ - MailmanService, Group, UWLDAPService, UWLDAPRecord + MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService from ceod.model.utils import strings_to_bytes +from .MockSMTPServer import MockSMTPServer +from .MockMailmanServer import MockMailmanServer +from .conftest_ceod_api import * -@pytest.fixture(autouse=True, scope='session') +@pytest.fixture(scope='session') def cfg(): - with importlib.resources.path('tests_common', 'ceod_test_local.ini') as p: + with importlib.resources.path('tests', 'ceod_test_local.ini') as p: config_file = p.__fspath__() _cfg = Config(config_file) component.provideUtility(_cfg, IConfig) return _cfg -@pytest.fixture(autouse=True, scope='session') +@pytest.fixture(scope='session') def krb_srv(cfg): # we need to be root to read the keytab assert os.geteuid() == 0 @@ -33,13 +37,12 @@ def krb_srv(cfg): principal = 'ceod/admin' else: principal = 'ceod/' + socket.getfqdn() - cache_file = '/tmp/ceod_test/krb5_cache' - if os.path.isfile(cache_file): - os.unlink(cache_file) - krb = KerberosService(principal, cache_file) + cache_dir = '/tmp/ceod_test/krb5_cache' + shutil.rmtree(cache_dir, ignore_errors=True) + krb = KerberosService(principal, cache_dir) component.provideUtility(krb, IKerberosService) yield krb - os.unlink(cache_file) + shutil.rmtree(cache_dir) def recursively_delete_subtree(conn: ldap.ldapobject.LDAPObject, base_dn: str): @@ -52,7 +55,7 @@ def recursively_delete_subtree(conn: ldap.ldapobject.LDAPObject, base_dn: str): pass -@pytest.fixture(autouse=True, scope='session') +@pytest.fixture(scope='session') def ldap_srv(cfg, krb_srv): conn = ldap.initialize(cfg.get('ldap_server_url')) conn.sasl_gssapi_bind_s() @@ -76,7 +79,7 @@ def ldap_srv(cfg, krb_srv): recursively_delete_subtree(conn, groups_base) -@pytest.fixture(autouse=True, scope='session') +@pytest.fixture(scope='session') def file_srv(cfg): _file_srv = FileService() component.provideUtility(_file_srv, IFileService) @@ -90,6 +93,82 @@ def file_srv(cfg): shutil.rmtree(clubs_home, ignore_errors=True) +@pytest.fixture(scope='session') +def http_client(cfg): + client = HTTPClient() + component.provideUtility(client, IHTTPClient) + return + + +@pytest.fixture(scope='session') +def mock_mailman_server(): + server = MockMailmanServer() + server.start() + yield server + server.stop() + + +@pytest.fixture(scope='session') +def mailman_srv(mock_mailman_server, cfg, http_client): + # TODO: test the RemoteMailmanService as well + mailman = MailmanService() + component.provideUtility(mailman, IMailmanService) + return mailman + + +@pytest.fixture(scope='session') +def uwldap_srv(cfg, ldap_srv): + conn = ldap.initialize(cfg.get('uwldap_server_url')) + conn.sasl_gssapi_bind_s() + base_dn = cfg.get('uwldap_base') + ou = base_dn.split(',', 1)[0].split('=')[1] + + recursively_delete_subtree(conn, base_dn) + + conn.add_s(base_dn, ldap.modlist.addModlist({ + 'objectClass': [b'organizationalUnit'], + 'ou': [ou.encode()] + })) + _uwldap_srv = UWLDAPService() + component.provideUtility(_uwldap_srv, IUWLDAPService) + yield _uwldap_srv + + recursively_delete_subtree(conn, base_dn) + + +@pytest.fixture(scope='session') +def mock_mail_server(): + mock_server = MockSMTPServer() + mock_server.start() + yield mock_server + mock_server.stop() + + +@pytest.fixture(scope='session') +def mail_srv(cfg, mock_mail_server): + _mail_srv = MailService() + component.provideUtility(_mail_srv, IMailService) + return _mail_srv + + +@pytest.fixture(autouse=True, scope='session') +def app( + cfg, + krb_srv, + ldap_srv, + file_srv, + mailman_srv, + uwldap_srv, + mail_srv, +): + # need to be root to read keytab + assert os.geteuid() == 0 + app = create_app({ + 'TESTING': True, + }) + return app + + @pytest.fixture def simple_user(): return User( @@ -123,24 +202,6 @@ def krb_user(simple_user): simple_user.remove_from_kerberos() -@pytest.fixture(scope='session') -def http_client(): - client = HTTPClient() - component.provideUtility(client, IHTTPClient) - return - - -@pytest.fixture(autouse=True, scope='session') -def mailman_srv(cfg, http_client): - if socket.gethostname() == cfg.get('ceod_mailman_host'): - # TODO: use a mock server on drone.io - mailman = MailmanService() - else: - mailman = RemoteMailmanService() - component.provideUtility(mailman, IMailmanService) - return mailman - - @pytest.fixture def simple_group(): return Group( @@ -156,26 +217,6 @@ def ldap_group(simple_group): simple_group.remove_from_ldap() -@pytest.fixture(scope='session') -def uwldap_srv(cfg, ldap_srv): - conn = ldap.initialize(cfg.get('uwldap_server_url')) - conn.sasl_gssapi_bind_s() - base_dn = cfg.get('uwldap_base') - ou = base_dn.split(',', 1)[0].split('=')[1] - - recursively_delete_subtree(conn, base_dn) - - conn.add_s(base_dn, ldap.modlist.addModlist({ - 'objectClass': [b'organizationalUnit'], - 'ou': [ou.encode()] - })) - _uwldap_srv = UWLDAPService() - component.provideUtility(_uwldap_srv, IUWLDAPService) - yield _uwldap_srv - - recursively_delete_subtree(conn, base_dn) - - @pytest.fixture def uwldap_user(cfg, uwldap_srv): conn = ldap.initialize(cfg.get('uwldap_server_url')) diff --git a/tests/conftest_ceod_api.py b/tests/conftest_ceod_api.py new file mode 100644 index 0000000..853da52 --- /dev/null +++ b/tests/conftest_ceod_api.py @@ -0,0 +1,66 @@ +import json +import socket + +from flask.testing import FlaskClient +import gssapi +import pytest +from requests import Request +from requests_gssapi import HTTPSPNEGOAuth +from zope import component + +from ceo_common.interfaces import IConfig + + +@pytest.fixture(scope='session') +def client(app): + app_client = app.test_client() + return CeodTestClient(app_client) + + +class CeodTestClient: + def __init__(self, app_client: FlaskClient): + cfg = component.getUtility(IConfig) + self.client = app_client + self.admin_principal = cfg.get('ldap_admin_principal') + # this is only used for the HTTPSNEGOAuth + self.base_url = f'http://{socket.getfqdn()}' + self.cached_auth = {} + + def get_auth(self, principal): + if principal in self.cached_auth: + return self.cached_auth[principal] + name = gssapi.Name(principal) + creds = gssapi.Credentials(name=name, usage='initiate') + auth = HTTPSPNEGOAuth( + opportunistic_auth=True, + target_name='ceod', + creds=creds, + ) + self.cached_auth[principal] = auth + return auth + + def get_headers(self, principal): + # method doesn't matter here because we just need the headers + req = Request('GET', self.base_url, auth=self.get_auth(principal)) + return req.prepare().headers.items() + + def request(self, method, path, principal, **kwargs): + if principal is None: + principal = self.admin_principal + resp = self.client.open( + path, method=method, headers=self.get_headers(principal), **kwargs) + status = int(resp.status.split(' ', 1)[0]) + try: + data = json.loads(resp.data) + except json.JSONDecodeError: + data = resp.data.decode() + return status, data + + def get(self, path, principal=None, **kwargs): + return self.request('GET', path, principal, **kwargs) + + def post(self, path, principal=None, **kwargs): + return self.request('POST', path, principal, **kwargs) + + def delete(self, path, principal=None, **kwargs): + return self.request('DELETE', path, principal, **kwargs)