move all tests to top-level folder

pull/5/head
Max Erenberg 1 year ago
parent cbf4aa43f8
commit 6cdb41d47b
  1. 81
      ceod/api/app_factory.py
  2. 7
      ceod/model/KerberosService.py
  3. 1
      ceod/model/test/conftest.py
  4. 2
      ceod/transactions/members/AddMemberTransaction.py
  5. 1
      clear_cache.sh
  6. 2
      dev-requirements.txt
  7. 49
      tests/MockMailmanServer.py
  8. 27
      tests/MockSMTPServer.py
  9. 2
      tests/__init__.py
  10. 0
      tests/ceod/api/__init__.py
  11. 23
      tests/ceod/api/test_members.py
  12. 0
      tests/ceod/model/__init__.py
  13. 0
      tests/ceod/model/test_group.py
  14. 10
      tests/ceod/model/test_mail.py
  15. 2
      tests/ceod/model/test_mailman.py
  16. 0
      tests/ceod/model/test_user.py
  17. 0
      tests/ceod/model/test_uwldap.py
  18. 0
      tests/ceod_dev.ini
  19. 2
      tests/ceod_test_local.ini
  20. 141
      tests/conftest.py
  21. 66
      tests/conftest_ceod_api.py

@ -18,67 +18,86 @@ def create_app(flask_config={}):
app = Flask(__name__)
app.config.from_mapping(flask_config)
if not app.config.get('TESTING'):
register_services(app)
cfg = component.getUtility(IConfig)
fqdn = socket.getfqdn()
os.environ['KRB5_KTNAME'] = '/etc/krb5.keytab'
init_kerberos(app, service='ceod', hostname=fqdn)
hostname = socket.gethostname()
# Only ceod_admin_host should serve the /api/members endpoints because
# it needs to run kadmin
if hostname == cfg.get('ceod_admin_host'):
from ceod.api import members
app.register_blueprint(members.bp, url_prefix='/api/members')
if hostname == cfg.get('ceod_mailman_host'):
# Only offer mailman API if this host is running Mailman
from ceod.api import mailman
app.register_blueprint(mailman.bp, url_prefix='/api/mailman')
from ceod.api import uwldap
app.register_blueprint(uwldap.bp, url_prefix='/api/uwldap')
register_error_handlers(app)
@app.route('/ping')
def ping():
"""Health check"""
return 'pong\n'
return app
def register_services(app):
# Config
if app.config.get('ENV') == 'development' and 'CEOD_CONFIG' not in os.environ:
with importlib.resources.path('tests_common', 'ceod_dev.ini') as p:
with importlib.resources.path('tests', 'ceod_dev.ini') as p:
config_file = p.__fspath__()
else:
config_file = None
cfg = Config(config_file)
component.provideUtility(cfg, IConfig)
init_kerberos(app, service='ceod')
# KerberosService
if 'KRB5_KTNAME' not in os.environ:
os.environ['KRB5_KTNAME'] = '/etc/krb5.keytab'
hostname = socket.gethostname()
fqdn = socket.getfqdn()
# Only ceod_admin_host has the ceod/admin key in its keytab
if hostname == cfg.get('ceod_admin_host'):
krb_srv = KerberosService(cfg.get('ldap_admin_principal'))
from ceod.api import members
app.register_blueprint(members.bp, url_prefix='/api/members')
principal = cfg.get('ldap_admin_principal')
else:
fqdn = socket.getfqdn()
krb_srv = KerberosService(f'ceod/{fqdn}')
principal = f'ceod/{fqdn}'
krb_srv = KerberosService(principal)
component.provideUtility(krb_srv, IKerberosService)
# Any host can use LDAPService, but only ceod_admin_host can write
# LDAPService
ldap_srv = LDAPService()
component.provideUtility(ldap_srv, ILDAPService)
# HTTPService
http_client = HTTPClient()
component.provideUtility(http_client, IHTTPClient)
# Only instantiate FileService if this host has NFS no_root_squash
# If admin_host and fs_root_host become separate, we will need
# to create a RemoteFileService
# FileService
if hostname == cfg.get('ceod_fs_root_host'):
file_srv = FileService()
component.provideUtility(file_srv, IFileService)
# Only offer mailman API if this host is running Mailman
# MailmanService
if hostname == cfg.get('ceod_mailman_host'):
mailman_srv = MailmanService()
component.provideUtility(mailman_srv, IMailmanService)
from ceod.api import mailman
app.register_blueprint(mailman.bp, url_prefix='/api/mailman')
else:
mailman_srv = RemoteMailmanService()
component.provideUtility(mailman_srv, IMailmanService)
component.provideUtility(mailman_srv, IMailmanService)
# MailService
mail_srv = MailService()
component.provideUtility(mail_srv, IMailService)
# UWLDAPService
uwldap_srv = UWLDAPService()
component.provideUtility(uwldap_srv, IUWLDAPService)
from ceod.api import uwldap
app.register_blueprint(uwldap.bp, url_prefix='/api/uwldap')
register_error_handlers(app)
@app.route('/ping')
def ping():
"""Health check"""
return 'pong\n'
return app

@ -11,11 +11,11 @@ class KerberosService:
def __init__(
self,
admin_principal: str,
cache_file: str = '/run/ceod/krb5_cache',
cache_dir: str = '/run/ceod/krb5_cache',
):
self.admin_principal = admin_principal
os.makedirs(os.path.dirname(cache_file), exist_ok=True)
os.putenv('KRB5CCNAME', 'FILE:' + cache_file)
os.makedirs(cache_dir, exist_ok=True)
os.environ['KRB5CCNAME'] = 'DIR:' + cache_dir
self.kinit()
def kinit(self):
@ -27,6 +27,7 @@ class KerberosService:
'-pw', password,
'-policy', 'default',
'+needchange',
'+requires_preauth',
principal
], check=True)

@ -1 +0,0 @@
from tests_common.fixtures import *

@ -44,7 +44,7 @@ class AddMemberTransaction(AbstractTransaction):
self.forwarding_addresses = forwarding_addresses
self.member = None
self.group = None
self.new_member_list = cfg.get('new_member_list')
self.new_member_list = cfg.get('mailman3_new_member_list')
self.mail_srv = component.getUtility(IMailService)
def child_execute_iter(self):

@ -1,2 +1,3 @@
#!/bin/sh
find ceo* -type d -name __pycache__ -execdir rm -r '{}' \;
rm -rf .pytest_cache

@ -2,3 +2,5 @@ flake8==3.9.2
setuptools==40.8.0
wheel==0.36.2
pytest==6.2.4
aiosmtpd==1.4.2
aiohttp==3.7.4.post0

@ -0,0 +1,49 @@
import asyncio
from threading import Thread
from aiohttp import web
class MockMailmanServer:
def __init__(self):
self.app = web.Application()
self.app.add_routes([
web.post('/members', self.subscribe),
web.delete('/lists/{mailing_list}/member/{address}', self.unsubscribe),
])
self.runner = web.AppRunner(self.app)
self.loop = asyncio.new_event_loop()
self.subscriptions = []
def _start_loop(self):
asyncio.set_event_loop(self.loop)
self.loop.run_until_complete(self.runner.setup())
site = web.TCPSite(self.runner, 'localhost', 8002)
self.loop.run_until_complete(site.start())
self.loop.run_forever()
def start(self):
t = Thread(target=self._start_loop)
t.start()
def stop(self):
self.loop.call_soon_threadsafe(self.loop.stop)
async def subscribe(self, request):
body = await request.post()
subscriber = body['subscriber']
if subscriber in self.subscriptions:
return web.json_response({
'description': 'user is already subscribed',
}, status=409)
self.subscriptions.append(subscriber)
return web.json_response({'status': 'OK'})
async def unsubscribe(self, request):
subscriber = request.match_info['address']
if subscriber not in self.subscriptions:
return web.json_response({
'description': 'user is not subscribed',
}, status=404)
self.subscriptions.remove(subscriber)
return web.json_response({'status': 'OK'})

@ -0,0 +1,27 @@
from aiosmtpd.controller import Controller
class MockSMTPServer:
def __init__(self, hostname='localhost', port=8025):
self.messages = []
self.controller = Controller(MockHandler(self), hostname, port)
def start(self):
self.controller.start()
def stop(self):
self.controller.stop()
class MockHandler:
def __init__(self, mock_server):
self.mock_server = mock_server
async def handle_DATA(self, server, session, envelope):
msg = {
'from': envelope.mail_from,
'to': envelope.rcpt_tos[0],
'content': envelope.content.decode(),
}
self.mock_server.messages.append(msg)
return '250 Message accepted for delivery'

@ -0,0 +1,2 @@
from .MockSMTPServer import MockSMTPServer
from .MockMailmanServer import MockMailmanServer

@ -0,0 +1,23 @@
import pytest
def test_members_get_user(client):
status, data = client.get('/api/members/no_such_user')
assert status == 404
assert data['error'] == 'user not found'
@pytest.fixture(scope='session')
def create_user_resp(client):
return client.post('/api/members', json={
'uid': 'test_jdoe',
'cn': 'John Doe',
'program': 'Math',
'terms': ['s2021'],
})
# def test_create_user(create_user_resp):
# status, data = create_user_resp
# assert status == 200
# # TODO: check response contents

@ -0,0 +1,10 @@
def test_welcome_message(cfg, mock_mail_server, mail_srv, simple_user):
base_domain = cfg.get('base_domain')
mail_srv.send_welcome_message_to(simple_user)
msg = mock_mail_server.messages[0]
assert msg['from'] == f'exec@{base_domain}'
assert msg['to'] == f'{simple_user.uid}@{base_domain}'
# make sure that templating was applied correctly
first_name = simple_user.cn.split()[0]
assert f'Hello {first_name}' in msg['content']
mock_mail_server.messages.clear()

@ -3,7 +3,7 @@ import pytest
from ceo_common.errors import UserAlreadySubscribedError, UserNotSubscribedError
def test_user_mailing_lists(ldap_user):
def test_user_mailing_lists(mailman_srv, ldap_user):
user = ldap_user
user.subscribe_to_mailing_list('csc-general')

@ -37,7 +37,7 @@ smtp_url = smtp://localhost:8025
smtp_starttls = false
[mailman3]
api_base_url = http://localhost:8001/3.1
api_base_url = http://localhost:8002
api_username = restadmin
api_password = mailman3
new_member_list = csc-general

@ -8,23 +8,27 @@ import socket
from zope import component
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
IFileService, IMailmanService, IHTTPClient, IUWLDAPService
IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService
from ceo_common.model import Config, RemoteMailmanService, HTTPClient
from ceod.api import create_app
from ceod.model import KerberosService, LDAPService, FileService, User, \
MailmanService, Group, UWLDAPService, UWLDAPRecord
MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService
from ceod.model.utils import strings_to_bytes
from .MockSMTPServer import MockSMTPServer
from .MockMailmanServer import MockMailmanServer
from .conftest_ceod_api import *
@pytest.fixture(autouse=True, scope='session')
@pytest.fixture(scope='session')
def cfg():
with importlib.resources.path('tests_common', 'ceod_test_local.ini') as p:
with importlib.resources.path('tests', 'ceod_test_local.ini') as p:
config_file = p.__fspath__()
_cfg = Config(config_file)
component.provideUtility(_cfg, IConfig)
return _cfg
@pytest.fixture(autouse=True, scope='session')
@pytest.fixture(scope='session')
def krb_srv(cfg):
# we need to be root to read the keytab
assert os.geteuid() == 0
@ -33,13 +37,12 @@ def krb_srv(cfg):
principal = 'ceod/admin'
else:
principal = 'ceod/' + socket.getfqdn()
cache_file = '/tmp/ceod_test/krb5_cache'
if os.path.isfile(cache_file):
os.unlink(cache_file)
krb = KerberosService(principal, cache_file)
cache_dir = '/tmp/ceod_test/krb5_cache'
shutil.rmtree(cache_dir, ignore_errors=True)
krb = KerberosService(principal, cache_dir)
component.provideUtility(krb, IKerberosService)
yield krb
os.unlink(cache_file)
shutil.rmtree(cache_dir)
def recursively_delete_subtree(conn: ldap.ldapobject.LDAPObject, base_dn: str):
@ -52,7 +55,7 @@ def recursively_delete_subtree(conn: ldap.ldapobject.LDAPObject, base_dn: str):
pass
@pytest.fixture(autouse=True, scope='session')
@pytest.fixture(scope='session')
def ldap_srv(cfg, krb_srv):
conn = ldap.initialize(cfg.get('ldap_server_url'))
conn.sasl_gssapi_bind_s()
@ -76,7 +79,7 @@ def ldap_srv(cfg, krb_srv):
recursively_delete_subtree(conn, groups_base)
@pytest.fixture(autouse=True, scope='session')
@pytest.fixture(scope='session')
def file_srv(cfg):
_file_srv = FileService()
component.provideUtility(_file_srv, IFileService)
@ -90,6 +93,82 @@ def file_srv(cfg):
shutil.rmtree(clubs_home, ignore_errors=True)
@pytest.fixture(scope='session')
def http_client(cfg):
client = HTTPClient()
component.provideUtility(client, IHTTPClient)
return
@pytest.fixture(scope='session')
def mock_mailman_server():
server = MockMailmanServer()
server.start()
yield server
server.stop()
@pytest.fixture(scope='session')
def mailman_srv(mock_mailman_server, cfg, http_client):
# TODO: test the RemoteMailmanService as well
mailman = MailmanService()
component.provideUtility(mailman, IMailmanService)
return mailman
@pytest.fixture(scope='session')
def uwldap_srv(cfg, ldap_srv):
conn = ldap.initialize(cfg.get('uwldap_server_url'))
conn.sasl_gssapi_bind_s()
base_dn = cfg.get('uwldap_base')
ou = base_dn.split(',', 1)[0].split('=')[1]
recursively_delete_subtree(conn, base_dn)
conn.add_s(base_dn, ldap.modlist.addModlist({
'objectClass': [b'organizationalUnit'],
'ou': [ou.encode()]
}))
_uwldap_srv = UWLDAPService()
component.provideUtility(_uwldap_srv, IUWLDAPService)
yield _uwldap_srv
recursively_delete_subtree(conn, base_dn)
@pytest.fixture(scope='session')
def mock_mail_server():
mock_server = MockSMTPServer()
mock_server.start()
yield mock_server
mock_server.stop()
@pytest.fixture(scope='session')
def mail_srv(cfg, mock_mail_server):
_mail_srv = MailService()
component.provideUtility(_mail_srv, IMailService)
return _mail_srv
@pytest.fixture(autouse=True, scope='session')
def app(
cfg,
krb_srv,
ldap_srv,
file_srv,
mailman_srv,
uwldap_srv,
mail_srv,
):
# need to be root to read keytab
assert os.geteuid() == 0
app = create_app({
'TESTING': True,
})
return app
@pytest.fixture
def simple_user():
return User(
@ -123,24 +202,6 @@ def krb_user(simple_user):
simple_user.remove_from_kerberos()
@pytest.fixture(scope='session')
def http_client():
client = HTTPClient()
component.provideUtility(client, IHTTPClient)
return
@pytest.fixture(autouse=True, scope='session')
def mailman_srv(cfg, http_client):
if socket.gethostname() == cfg.get('ceod_mailman_host'):
# TODO: use a mock server on drone.io
mailman = MailmanService()
else:
mailman = RemoteMailmanService()
component.provideUtility(mailman, IMailmanService)
return mailman
@pytest.fixture
def simple_group():
return Group(
@ -156,26 +217,6 @@ def ldap_group(simple_group):
simple_group.remove_from_ldap()
@pytest.fixture(scope='session')
def uwldap_srv(cfg, ldap_srv):
conn = ldap.initialize(cfg.get('uwldap_server_url'))
conn.sasl_gssapi_bind_s()
base_dn = cfg.get('uwldap_base')
ou = base_dn.split(',', 1)[0].split('=')[1]
recursively_delete_subtree(conn, base_dn)
conn.add_s(base_dn, ldap.modlist.addModlist({
'objectClass': [b'organizationalUnit'],
'ou': [ou.encode()]
}))
_uwldap_srv = UWLDAPService()
component.provideUtility(_uwldap_srv, IUWLDAPService)
yield _uwldap_srv
recursively_delete_subtree(conn, base_dn)
@pytest.fixture
def uwldap_user(cfg, uwldap_srv):
conn = ldap.initialize(cfg.get('uwldap_server_url'))

@ -0,0 +1,66 @@
import json
import socket
from flask.testing import FlaskClient
import gssapi
import pytest
from requests import Request
from requests_gssapi import HTTPSPNEGOAuth
from zope import component
from ceo_common.interfaces import IConfig
@pytest.fixture(scope='session')
def client(app):
app_client = app.test_client()
return CeodTestClient(app_client)
class CeodTestClient:
def __init__(self, app_client: FlaskClient):
cfg = component.getUtility(IConfig)
self.client = app_client
self.admin_principal = cfg.get('ldap_admin_principal')
# this is only used for the HTTPSNEGOAuth
self.base_url = f'http://{socket.getfqdn()}'
self.cached_auth = {}
def get_auth(self, principal):
if principal in self.cached_auth:
return self.cached_auth[principal]
name = gssapi.Name(principal)
creds = gssapi.Credentials(name=name, usage='initiate')
auth = HTTPSPNEGOAuth(
opportunistic_auth=True,
target_name='ceod',
creds=creds,
)
self.cached_auth[principal] = auth
return auth
def get_headers(self, principal):
# method doesn't matter here because we just need the headers
req = Request('GET', self.base_url, auth=self.get_auth(principal))
return req.prepare().headers.items()
def request(self, method, path, principal, **kwargs):
if principal is None:
principal = self.admin_principal
resp = self.client.open(
path, method=method, headers=self.get_headers(principal), **kwargs)
status = int(resp.status.split(' ', 1)[0])
try:
data = json.loads(resp.data)
except json.JSONDecodeError:
data = resp.data.decode()
return status, data
def get(self, path, principal=None, **kwargs):
return self.request('GET', path, principal, **kwargs)
def post(self, path, principal=None, **kwargs):
return self.request('POST', path, principal, **kwargs)
def delete(self, path, principal=None, **kwargs):
return self.request('DELETE', path, principal, **kwargs)
Loading…
Cancel
Save