From 1e94132e9701c731dc47ca5e4efe75cefbe2a60e Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Sat, 1 Jan 2022 00:49:05 -0500 Subject: [PATCH] Add container registry API (#42) Add an API for members to create a project on Harbor. Co-authored-by: Max Erenberg <> Reviewed-on: https://git.csclub.uwaterloo.ca/public/pyceo/pulls/42 Co-authored-by: Max Erenberg Co-committed-by: Max Erenberg --- .drone/mail-setup.sh | 1 + ceo/cli/entrypoint.py | 2 + ceo/cli/registry.py | 21 ++ .../interfaces/IContainerRegistryService.py | 22 ++ ceo_common/interfaces/IUser.py | 21 +- ceo_common/interfaces/__init__.py | 1 + ceod/api/app_factory.py | 11 +- ceod/api/cloud.py | 11 +- ceod/model/CloudResourceManager.py | 201 ++++++++++++------ ceod/model/ContainerRegistryService.py | 87 ++++++++ ceod/model/User.py | 18 +- ceod/model/__init__.py | 1 + etc/ceod.ini | 5 + tests/MockHarborServer.py | 104 +++++++++ tests/ceo/cli/test_cloud.py | 12 ++ tests/ceod/api/test_cloud.py | 16 +- tests/ceod/model/test_container_registry.py | 24 +++ tests/ceod_dev.ini | 5 + tests/ceod_test_local.ini | 5 + tests/conftest.py | 22 +- 20 files changed, 515 insertions(+), 75 deletions(-) create mode 100644 ceo/cli/registry.py create mode 100644 ceo_common/interfaces/IContainerRegistryService.py create mode 100644 ceod/model/ContainerRegistryService.py create mode 100644 tests/MockHarborServer.py create mode 100644 tests/ceod/model/test_container_registry.py diff --git a/.drone/mail-setup.sh b/.drone/mail-setup.sh index 0265936..2751736 100755 --- a/.drone/mail-setup.sh +++ b/.drone/mail-setup.sh @@ -12,6 +12,7 @@ add_fqdn_to_hosts $(get_ip_addr auth1) auth1 python -m tests.MockMailmanServer & python -m tests.MockSMTPServer & python -m tests.MockCloudStackServer & +python -m tests.MockHarborServer & export DEBIAN_FRONTEND=noninteractive apt update diff --git a/ceo/cli/entrypoint.py b/ceo/cli/entrypoint.py index 3b825b2..0f680b1 100644 --- a/ceo/cli/entrypoint.py +++ b/ceo/cli/entrypoint.py @@ -9,6 +9,7 @@ from .postgresql import postgresql from .mailman import mailman from .cloud import cloud from .k8s import k8s +from .registry import registry @click.group() @@ -25,3 +26,4 @@ cli.add_command(postgresql) cli.add_command(mailman) cli.add_command(cloud) cli.add_command(k8s) +cli.add_command(registry) diff --git a/ceo/cli/registry.py b/ceo/cli/registry.py new file mode 100644 index 0000000..dddd960 --- /dev/null +++ b/ceo/cli/registry.py @@ -0,0 +1,21 @@ +import click + +from ..utils import http_post +from .utils import handle_sync_response + + +@click.group(short_help='Manage your container registry account') +def registry(): + pass + + +@registry.group(short_help='Manage your container registry project') +def project(): + pass + + +@project.command(short_help='Create a registry project') +def create(): + resp = http_post('/api/cloud/registry/projects') + handle_sync_response(resp) + click.echo('Congratulations! Your registry project was successfully created.') diff --git a/ceo_common/interfaces/IContainerRegistryService.py b/ceo_common/interfaces/IContainerRegistryService.py new file mode 100644 index 0000000..915bf04 --- /dev/null +++ b/ceo_common/interfaces/IContainerRegistryService.py @@ -0,0 +1,22 @@ +from typing import List + +from zope.interface import Interface + + +class IContainerRegistryService(Interface): + """Manage Harbor projects and users.""" + + def get_accounts() -> List[str]: + """Get a list of Harbor account usernames.""" + + def create_project_for_user(username: str): + """ + Create a new Harbor project for a user add make them a Project Admin. + The user needs to have logged in to Harbor at least once. + """ + + def delete_project_for_user(username: str): + """ + Deletes the Harbor project for the given user, if it exists. + All repositories in the project will be deleted. + """ diff --git a/ceo_common/interfaces/IUser.py b/ceo_common/interfaces/IUser.py index 7523354..a5ebe46 100644 --- a/ceo_common/interfaces/IUser.py +++ b/ceo_common/interfaces/IUser.py @@ -4,7 +4,18 @@ from zope.interface import Interface, Attribute class IUser(Interface): - """Represents a Unix user.""" + """ + Represents a Unix user. + + There are four types of Unix users in the CSC LDAP: + 1. Members + 2. Club reps + 3. Clubs + 4. System accounts (e.g. syscom, sysadmin, progcom, exec, git, www) + + Members can become club reps and vice versa. The last term registration + of a user determines if they are a member or club rep. + """ # LDAP attributes uid = Attribute('user identifier') @@ -39,6 +50,14 @@ class IUser(Interface): Returns False if this is the Unix user for a member. """ + def is_member_or_club_rep() -> bool: + """ + Returns True iff this user has the 'member' objectClass. + """ + + def is_member() -> bool: + """Returns True iff this user is a member.""" + def add_to_ldap(): """ Add a new record to LDAP for this user. diff --git a/ceo_common/interfaces/__init__.py b/ceo_common/interfaces/__init__.py index 72c9090..25c0ffd 100644 --- a/ceo_common/interfaces/__init__.py +++ b/ceo_common/interfaces/__init__.py @@ -13,3 +13,4 @@ from .IHTTPClient import IHTTPClient from .IDatabaseService import IDatabaseService from .IVHostManager import IVHostManager from .IKubernetesService import IKubernetesService +from .IContainerRegistryService import IContainerRegistryService diff --git a/ceod/api/app_factory.py b/ceod/api/app_factory.py index 9c5f1a4..98d30eb 100644 --- a/ceod/api/app_factory.py +++ b/ceod/api/app_factory.py @@ -8,12 +8,14 @@ from zope import component from .error_handlers import register_error_handlers from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \ IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \ - ICloudStackService, ICloudResourceManager, IKubernetesService, IVHostManager + ICloudStackService, ICloudResourceManager, IKubernetesService, IVHostManager, \ + IContainerRegistryService from ceo_common.model import Config, HTTPClient, RemoteMailmanService from ceod.api.spnego import init_spnego from ceod.model import KerberosService, LDAPService, FileService, \ MailmanService, MailService, UWLDAPService, CloudStackService, \ - CloudResourceManager, KubernetesService, VHostManager + CloudResourceManager, KubernetesService, VHostManager, \ + ContainerRegistryService from ceod.db import MySQLService, PostgreSQLService @@ -123,7 +125,7 @@ def register_services(app): psql_srv = PostgreSQLService() component.provideUtility(psql_srv, IDatabaseService, 'postgresql') - # CloudStackService, CloudResourceManager, VHostManager, KubernetesService + # all of the cloud services if hostname == cfg.get('ceod_cloud_host'): cloudstack_srv = CloudStackService() component.provideUtility(cloudstack_srv, ICloudStackService) @@ -136,3 +138,6 @@ def register_services(app): k8s_srv = KubernetesService() component.provideUtility(k8s_srv, IKubernetesService) + + reg_srv = ContainerRegistryService() + component.provideUtility(reg_srv, IContainerRegistryService) diff --git a/ceod/api/cloud.py b/ceod/api/cloud.py index d91b790..9d89389 100644 --- a/ceod/api/cloud.py +++ b/ceod/api/cloud.py @@ -4,7 +4,7 @@ from zope import component from .utils import requires_authentication_no_realm, authz_restrict_to_syscom, \ get_valid_member_or_throw from ceo_common.interfaces import ICloudStackService, IVHostManager, \ - IKubernetesService, ICloudResourceManager + IKubernetesService, ICloudResourceManager, IContainerRegistryService bp = Blueprint('cloud', __name__) @@ -62,3 +62,12 @@ def create_k8s_account(auth_user: str): 'status': 'OK', 'kubeconfig': kubeconfig, } + + +@bp.route('/registry/projects', methods=['POST']) +@requires_authentication_no_realm +def create_registry_project(auth_user: str): + get_valid_member_or_throw(auth_user) + reg_srv = component.getUtility(IContainerRegistryService) + reg_srv.create_project_for_user(auth_user) + return {'status': 'OK'} diff --git a/ceod/model/CloudResourceManager.py b/ceod/model/CloudResourceManager.py index 9289ea1..00c830a 100644 --- a/ceod/model/CloudResourceManager.py +++ b/ceod/model/CloudResourceManager.py @@ -2,15 +2,16 @@ from collections import defaultdict import datetime import json import os -from typing import Dict +from typing import Dict, List from zope import component from zope.interface import implementer +from ceo_common.errors import UserNotFoundError from ceo_common.logger_factory import logger_factory -from ceo_common.interfaces import ICloudResourceManager, \ +from ceo_common.interfaces import ICloudResourceManager, IUser, \ ILDAPService, IMailService, IKubernetesService, IVHostManager, \ - ICloudStackService + ICloudStackService, IContainerRegistryService from ceo_common.model import Term import ceo_common.utils as utils @@ -26,79 +27,133 @@ class CloudResourceManager: self.pending_deletions_file = \ os.path.join(state_dir, 'pending_account_deletions.json') - def purge_accounts(self) -> Dict: - accounts_deleted = [] - accounts_to_be_deleted = [] - result = { - 'accounts_deleted': accounts_deleted, - 'accounts_to_be_deleted': accounts_to_be_deleted, + @staticmethod + def _should_not_have_resources_deleted(user: IUser) -> bool: + return not user.is_member() or user.membership_is_valid() + + def _get_resources_for_each_user(self) -> Dict[str, Dict]: + """ + Get a list of cloud resources each user is using. + The returned dict looks like + { + "ctdalek": { + "resources": ["cloudstack", "k8s", ...], + "cloudstack_account_id": "3452345-2453245-23453..." + }, + ... } + The "cloudstack_account_id" key will only be present if the user + has a CloudStack account. + """ + k8s_srv = component.getUtility(IKubernetesService) + vhost_mgr = component.getUtility(IVHostManager) + cloudstack_srv = component.getUtility(ICloudStackService) + reg_srv = component.getUtility(IContainerRegistryService) - current_term = Term.current() - beginning_of_term = current_term.to_datetime() - now = utils.get_current_datetime() - delta = now - beginning_of_term - if delta.days < 30: - # one-month grace period - return result + accounts = defaultdict(lambda: {'resources': []}) + cloudstack_accounts = cloudstack_srv.get_accounts() + # note that cloudstack_accounts is a dict, not a list + for username, account_id in cloudstack_accounts.items(): + accounts[username]['resources'].append('cloudstack') + accounts[username]['cloudstack_account_id'] = account_id + + vhost_accounts = vhost_mgr.get_accounts() + for username in vhost_accounts: + accounts[username]['resources'].append('vhost') + + k8s_accounts = k8s_srv.get_accounts() + for username in k8s_accounts: + accounts[username]['resources'].append('k8s') + + reg_accounts = reg_srv.get_accounts() + for username in reg_accounts: + accounts[username]['resources'].append('registry') + + return accounts + + def _perform_deletions( + self, + state: Dict, + accounts: Dict[str, Dict], + accounts_deleted: List[str], + ): ldap_srv = component.getUtility(ILDAPService) mail_srv = component.getUtility(IMailService) k8s_srv = component.getUtility(IKubernetesService) vhost_mgr = component.getUtility(IVHostManager) cloudstack_srv = component.getUtility(ICloudStackService) + reg_srv = component.getUtility(IContainerRegistryService) - # get a list of all cloud services each user is using - accounts = defaultdict(list) - cloudstack_accounts = cloudstack_srv.get_accounts() - # note that cloudstack_accounts is a dict, not a list - for username in cloudstack_accounts: - accounts[username].append('cloudstack') - vhost_accounts = vhost_mgr.get_accounts() - for username in vhost_accounts: - accounts[username].append('vhost') - k8s_accounts = k8s_srv.get_accounts() - for username in k8s_accounts: - accounts[username].append('k8s') - - if os.path.isfile(self.pending_deletions_file): - state = json.load(open(self.pending_deletions_file)) - last_check = datetime.datetime.fromtimestamp(state['timestamp']) - delta = now - last_check - if delta.days < 7: - logger.debug( - 'Skipping account purge because less than one week has ' - 'passed since the warning emails were sent out' - ) - accounts_to_be_deleted.extend(state['accounts_to_be_deleted']) - return result - for username in state['accounts_to_be_deleted']: - if username not in accounts: - continue + for username in state['accounts_to_be_deleted']: + if username not in accounts: + continue + try: user = ldap_srv.get_user(username) - if user.membership_is_valid(): - continue - services = accounts[username] - if 'cloudstack' in services: - account_id = cloudstack_accounts[username] - cloudstack_srv.delete_account(account_id) - if 'vhost' in services: - vhost_mgr.delete_all_vhosts_for_user(username) - if 'k8s' in services: - k8s_srv.delete_account(username) - accounts_deleted.append(username) - mail_srv.send_cloud_account_has_been_deleted_message(user) - logger.info(f'Deleted cloud resources for {username}') - os.unlink(self.pending_deletions_file) - return result + except UserNotFoundError: + continue + if self._should_not_have_resources_deleted(user): + continue + resources = accounts[username]['resources'] + if 'cloudstack' in resources: + account_id = accounts[username]['cloudstack_account_id'] + cloudstack_srv.delete_account(account_id) + if 'vhost' in resources: + vhost_mgr.delete_all_vhosts_for_user(username) + if 'k8s' in resources: + k8s_srv.delete_account(username) + if 'registry' in resources: + reg_srv.delete_project_for_user(username) + accounts_deleted.append(username) + mail_srv.send_cloud_account_has_been_deleted_message(user) + logger.info(f'Deleted cloud resources for {username}') + def _perform_deletions_if_warning_period_passed( + self, + now: datetime.datetime, + accounts: Dict[str, Dict], + accounts_deleted: List[str], + accounts_to_be_deleted: List[str], + ): + state = json.load(open(self.pending_deletions_file)) + last_check = datetime.datetime.fromtimestamp(state['timestamp']) + delta = now - last_check + if delta.days < 7: + logger.debug( + 'Skipping account purge because less than one week has ' + 'passed since the warning emails were sent out' + ) + accounts_to_be_deleted.extend(state['accounts_to_be_deleted']) + return + self._perform_deletions(state, accounts, accounts_deleted) + os.unlink(self.pending_deletions_file) + + def _in_grace_period(self, now: datetime.datetime) -> bool: + current_term = Term.current() + beginning_of_term = current_term.to_datetime() + delta = now - beginning_of_term + # one-month grace period + return delta.days < 30 + + def _send_out_warning_emails( + self, + now: datetime.datetime, + accounts: Dict[str, dict], + accounts_to_be_deleted: List[str], + ): + ldap_srv = component.getUtility(ILDAPService) + mail_srv = component.getUtility(IMailService) state = { 'timestamp': int(now.timestamp()), 'accounts_to_be_deleted': accounts_to_be_deleted, } for username in accounts: - user = ldap_srv.get_user(username) - if user.membership_is_valid(): + try: + user = ldap_srv.get_user(username) + except UserNotFoundError: + logger.warning(f'User {username} not found') + continue + if self._should_not_have_resources_deleted(user): continue accounts_to_be_deleted.append(username) mail_srv.send_cloud_account_will_be_deleted_message(user) @@ -108,4 +163,30 @@ class CloudResourceManager: ) if accounts_to_be_deleted: json.dump(state, open(self.pending_deletions_file, 'w')) + + def _warning_emails_were_sent_out(self) -> bool: + return os.path.isfile(self.pending_deletions_file) + + def purge_accounts(self) -> Dict: + accounts_deleted = [] + accounts_to_be_deleted = [] + result = { + 'accounts_deleted': accounts_deleted, + 'accounts_to_be_deleted': accounts_to_be_deleted, + } + + now = utils.get_current_datetime() + if self._in_grace_period(now): + return result + + # get a list of all cloud services each user is using + accounts = self._get_resources_for_each_user() + + if self._warning_emails_were_sent_out(): + self._perform_deletions_if_warning_period_passed( + now, accounts, accounts_deleted, accounts_to_be_deleted) + return result + + self._send_out_warning_emails(now, accounts, accounts_to_be_deleted) + return result diff --git a/ceod/model/ContainerRegistryService.py b/ceod/model/ContainerRegistryService.py new file mode 100644 index 0000000..c921905 --- /dev/null +++ b/ceod/model/ContainerRegistryService.py @@ -0,0 +1,87 @@ +from typing import List + +import requests +from requests.auth import HTTPBasicAuth +from zope import component +from zope.interface import implementer + +from ceo_common.errors import UserNotFoundError +from ceo_common.interfaces import IContainerRegistryService, IConfig + + +@implementer(IContainerRegistryService) +class ContainerRegistryService: + def __init__(self): + cfg = component.getUtility(IConfig) + self.base_url = cfg.get('registry_base_url') + if self.base_url.endswith('/'): + self.base_url = self.base_url[:-1] + api_username = cfg.get('registry_username') + api_password = cfg.get('registry_password') + self.basic_auth = HTTPBasicAuth(api_username, api_password) + + def _http_request(self, method: str, path: str, **kwargs): + return requests.request( + method, self.base_url + path, **kwargs, auth=self.basic_auth) + + def _http_get(self, path: str, **kwargs): + return self._http_request('GET', path, **kwargs) + + def _http_post(self, path, **kwargs): + return self._http_request('POST', path, **kwargs) + + def _http_delete(self, path, **kwargs): + return self._http_request('DELETE', path, **kwargs) + + def _get_account(self, username: str): + resp = self._http_get('/users', params={'username': username}) + users = resp.json() + if len(users) < 1: + raise UserNotFoundError(username) + return users[0] + + def get_accounts(self) -> List[str]: + # We're only interested in accounts which have a project, so + # we're actually just going to get a list of projects + resp = self._http_get('/projects') + resp.raise_for_status() + # This project is only owned by the admin account, so we don't + # want to include it + project_exceptions = ['library'] + return [ + project['name'] for project in resp.json() + if project['name'] not in project_exceptions + ] + + def create_project_for_user(self, username: str): + user_id = self._get_account(username)['user_id'] + + # Create the project + resp = self._http_post( + '/projects', json={'project_name': username, 'public': True}) + # 409 => project already exists (that is OK) + if resp.status_code != 409: + resp.raise_for_status() + + # Add the user as a project admin (role ID 1) + resp = self._http_post( + f'/projects/{username}/members', + json={'role_id': 1, 'member_user': {'user_id': user_id}}) + # 409 => project member already exists (that is OK) + if resp.status_code != 409: + resp.raise_for_status() + + def delete_project_for_user(self, username: str): + # Delete all of the repositories inside the project first + resp = self._http_get(f'/projects/{username}/repositories') + if resp.status_code == 403: + # For some reason a 403 is returned if the project doesn't exist + return + resp.raise_for_status() + repositories = [repo['name'] for repo in resp.json()] + for repo in repositories: + resp = self._http_delete(f'/projects/{username}/repositories/{repo}') + resp.raise_for_status() + # Delete the project now that it is empty + resp = self._http_delete(f'/projects/{username}') + resp.raise_for_status() diff --git a/ceod/model/User.py b/ceod/model/User.py index 7118376..184b511 100644 --- a/ceod/model/User.py +++ b/ceod/model/User.py @@ -32,6 +32,7 @@ class User: mail_local_addresses: Union[List[str], None] = None, is_club_rep: Union[bool, None] = None, is_club: bool = False, + is_member_or_club_rep: Union[bool, None] = None, ldap3_entry: Union[ldap3.Entry, None] = None, shadowExpire: Union[int, None] = None, ): @@ -61,11 +62,13 @@ class User: if is_club_rep is None: if is_club: # not a real user - self.is_club_rep = False + is_club_rep = False else: - self.is_club_rep = should_be_club_rep(terms, non_member_terms) - else: - self.is_club_rep = is_club_rep + is_club_rep = should_be_club_rep(terms, non_member_terms) + self.is_club_rep = is_club_rep + if is_member_or_club_rep is None: + is_member_or_club_rep = terms is not None or non_member_terms is not None + self._is_member_or_club_rep = is_member_or_club_rep self.ldap3_entry = ldap3_entry self.shadowExpire = shadowExpire @@ -107,6 +110,12 @@ class User: def is_club(self) -> bool: return self._is_club + def is_member_or_club_rep(self) -> bool: + return self._is_member_or_club_rep + + def is_member(self): + return self.is_member_or_club_rep() and not self.is_club_rep + def add_to_ldap(self): if not self.mail_local_addresses: self.mail_local_addresses = [f'{self.uid}@{self.base_domain}'] @@ -158,6 +167,7 @@ class User: mail_local_addresses=attrs.get('mailLocalAddress'), is_club_rep=attrs.get('isClubRep', [False])[0], is_club=('club' in attrs['objectClass']), + is_member_or_club_rep=('member' in attrs['objectClass']), shadowExpire=attrs.get('shadowExpire'), ldap3_entry=entry, ) diff --git a/ceod/model/__init__.py b/ceod/model/__init__.py index 4e36718..a8b16ec 100644 --- a/ceod/model/__init__.py +++ b/ceod/model/__init__.py @@ -12,3 +12,4 @@ from .MailService import MailService from .MailmanService import MailmanService from .VHostManager import VHostManager from .KubernetesService import KubernetesService +from .ContainerRegistryService import ContainerRegistryService diff --git a/etc/ceod.ini b/etc/ceod.ini index 8313285..fe48955 100644 --- a/etc/ceod.ini +++ b/etc/ceod.ini @@ -100,3 +100,8 @@ members_clusterrole = csc-members-default members_group = csc-members authority_cert_path = /etc/csc/k8s-authority.crt server_url = https://172.19.134.149:6443 + +[registry] +base_url = https://registry.cloud.csclub.uwaterloo.ca/api/v2.0 +username = REPLACE_ME +password = REPLACE_ME diff --git a/tests/MockHarborServer.py b/tests/MockHarborServer.py new file mode 100644 index 0000000..87cf03a --- /dev/null +++ b/tests/MockHarborServer.py @@ -0,0 +1,104 @@ +from aiohttp import web + +from .MockHTTPServerBase import MockHTTPServerBase + + +class MockHarborServer(MockHTTPServerBase): + def __init__(self, port=8002): + prefix = '/api/v2.0' + routes = [ + web.get(prefix + '/users', self.users_get_handler), + web.get(prefix + '/projects', self.projects_get_handler), + web.post(prefix + '/projects', self.projects_post_handler), + web.post(prefix + '/projects/{project}/members', self.members_post_handler), + web.get(prefix + '/projects/{project}/repositories', self.repositories_get_handler), + web.delete(prefix + '/projects/{project}/repositories/{repository}', self.repositories_delete_handler), + web.delete(prefix + '/projects/{project}', self.projects_delete_handler), + # for debugging purposes + web.post('/reset', self.reset_handler), + web.delete('/users/{username}', self.users_delete_handler), + ] + super().__init__(port, routes) + + self.users = ['ctdalek', 'regular1', 'exec1'] + self.projects = { + 'ctdalek': ['repo1', 'repo2'], + 'regular1': [], + 'exec1': [], + } + + async def projects_get_handler(self, request): + return web.json_response([ + {'name': name, 'project_id': i + 1} + for i, name in enumerate(self.projects.keys()) + ]) + + async def projects_delete_handler(self, request): + project_name = request.match_info['project'] + if project_name not in self.projects: + return web.json_response({"errors": [{ + "code": "FORBIDDEN", "message": "forbidden" + }]}, status=403) + del self.projects[project_name] + return web.Response(text='', status=200) + + async def repositories_delete_handler(self, request): + project_name = request.match_info['project'] + repository_name = request.match_info['repository'] + self.projects[project_name].remove(repository_name) + return web.Response(text='', status=200) + + async def repositories_get_handler(self, request): + project_name = request.match_info['project'] + if project_name not in self.projects: + return web.json_response({"errors": [{ + "code": "FORBIDDEN", "message": "forbidden" + }]}, status=403) + projects = self.projects[project_name] + return web.json_response([ + {'id': i, 'name': name} for i, name in enumerate(projects) + ]) + + async def users_get_handler(self, request): + username = request.query['username'] + if username not in self.users: + return web.json_response([]) + return web.json_response([{ + 'username': username, + 'realname': username, + 'user_id': self.users.index(username), + 'email': username + '@csclub.internal', + }]) + + async def members_post_handler(self, request): + await request.json() + return web.Response(text='', status=201) + + async def projects_post_handler(self, request): + body = await request.json() + project_name = body['project_name'] + if project_name in self.projects: + return web.json_response({'errors': [{ + "code": "CONFLICT", + "message": f"The project named {project_name} already exists", + }]}, status=409) + self.projects[project_name] = ['repo1', 'repo2'] + return web.Response(text='', status=201) + + async def users_delete_handler(self, request): + username = request.match_info['username'] + self.users.remove(username) + return web.Response(text='OK\n', status=201) + + def reset(self): + self.users.clear() + self.projects.clear() + + async def reset_handler(self, request): + self.reset() + return web.Response(text='OK\n') + + +if __name__ == '__main__': + server = MockHarborServer() + server.start() diff --git a/tests/ceo/cli/test_cloud.py b/tests/ceo/cli/test_cloud.py index 465f1ca..4ab3b36 100644 --- a/tests/ceo/cli/test_cloud.py +++ b/tests/ceo/cli/test_cloud.py @@ -76,3 +76,15 @@ def test_k8s_account_activate(cli_setup, new_user): assert result.exit_code == 0 assert result.output == expected assert os.path.isfile(os.path.join(new_user.home_directory, '.kube', 'config')) + + +def test_registry_project_create(cli_setup, mock_harbor_server, new_user): + uid = new_user.uid + runner = CliRunner() + mock_harbor_server.reset() + mock_harbor_server.users.append(uid) + with gssapi_token_ctx(uid): + result = runner.invoke(cli, ['registry', 'project', 'create']) + expected = 'Congratulations! Your registry project was successfully created.\n' + assert result.exit_code == 0 + assert result.output == expected diff --git a/tests/ceod/api/test_cloud.py b/tests/ceod/api/test_cloud.py index 9c4e39e..c91fb56 100644 --- a/tests/ceod/api/test_cloud.py +++ b/tests/ceod/api/test_cloud.py @@ -35,8 +35,8 @@ def test_create_account(client, mock_cloud_server, new_user, ldap_conn): def test_purge_accounts( - client, mock_cloud_server, cloud_mgr, mock_mail_server, new_user, - ldap_conn, + client, mock_cloud_server, cloud_mgr, mock_mail_server, + mock_harbor_server, new_user, ldap_conn, ): uid = new_user.uid mock_cloud_server.clear() @@ -167,12 +167,14 @@ def test_cloud_vhosts(cfg, client, new_user, ldap_conn): assert status == 403 -def test_cloud_vhosts_purged_account( - cfg, client, mock_cloud_server, mock_mail_server, new_user, ldap_conn, +def test_cloud_resources_purged_account( + cfg, client, mock_cloud_server, mock_mail_server, mock_harbor_server, + new_user, ldap_conn, ): uid = new_user.uid members_domain = cfg.get('cloud vhosts_members_domain') mock_cloud_server.clear() + mock_harbor_server.reset() current_term = Term.current() beginning_of_term = current_term.to_datetime() domain1 = uid + '.' + members_domain @@ -182,6 +184,9 @@ def test_cloud_vhosts_purged_account( client.put( f'/api/cloud/vhosts/{domain1}', json={'ip_address': ip1}, principal=uid) + mock_harbor_server.users.append(uid) + client.post('/api/cloud/registry/projects', principal=uid) + assert len(mock_harbor_server.projects) == 1 expire_member(new_user, ldap_conn) with patch.object(ceo_common_utils, 'get_current_datetime') as now_mock: @@ -198,6 +203,9 @@ def test_cloud_vhosts_purged_account( assert status == 200 assert data == {'vhosts': []} + # registry project should have been deleted + assert len(mock_harbor_server.projects) == 0 + mock_mail_server.messages.clear() diff --git a/tests/ceod/model/test_container_registry.py b/tests/ceod/model/test_container_registry.py new file mode 100644 index 0000000..801f5ec --- /dev/null +++ b/tests/ceod/model/test_container_registry.py @@ -0,0 +1,24 @@ +import pytest + +from ceo_common.errors import UserNotFoundError + + +def test_registry(mock_harbor_server, registry_srv): + mock_harbor_server.reset() + username = 'test1' + + with pytest.raises(UserNotFoundError): + registry_srv.create_project_for_user(username) + + mock_harbor_server.users.append(username) + registry_srv.create_project_for_user(username) + assert username in mock_harbor_server.projects + # trying to create a project with the same name should have no effect + registry_srv.create_project_for_user(username) + + assert registry_srv.get_accounts() == [username] + + registry_srv.delete_project_for_user(username) + assert username not in mock_harbor_server.projects + # trying to delete a nonexistent project should have no effect + registry_srv.delete_project_for_user(username) diff --git a/tests/ceod_dev.ini b/tests/ceod_dev.ini index 3f360b9..1401700 100644 --- a/tests/ceod_dev.ini +++ b/tests/ceod_dev.ini @@ -94,3 +94,8 @@ members_clusterrole = csc-members-default members_group = csc-members authority_cert_path = /etc/csc/k8s-authority.crt server_url = https://172.19.134.149:6443 + +[registry] +base_url = http://localhost:8002/api/v2.0 +username = REPLACE_ME +password = REPLACE_ME diff --git a/tests/ceod_test_local.ini b/tests/ceod_test_local.ini index 972780d..2102bdb 100644 --- a/tests/ceod_test_local.ini +++ b/tests/ceod_test_local.ini @@ -93,3 +93,8 @@ members_clusterrole = csc-members-default members_group = csc-members authority_cert_path = /etc/csc/k8s-authority.crt server_url = https://172.19.134.149:6443 + +[registry] +base_url = http://localhost:8002/api/v2.0 +username = REPLACE_ME +password = REPLACE_ME diff --git a/tests/conftest.py b/tests/conftest.py index 039f313..26817e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,16 +26,18 @@ from .utils import ( # noqa: F401 from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \ IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \ IDatabaseService, ICloudStackService, IKubernetesService, IVHostManager, \ - ICloudResourceManager + ICloudResourceManager, IContainerRegistryService from ceo_common.model import Config, HTTPClient, Term from ceod.api import create_app from ceod.db import MySQLService, PostgreSQLService from ceod.model import KerberosService, LDAPService, FileService, User, \ MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \ - CloudStackService, KubernetesService, VHostManager, CloudResourceManager + CloudStackService, KubernetesService, VHostManager, CloudResourceManager, \ + ContainerRegistryService from .MockSMTPServer import MockSMTPServer from .MockMailmanServer import MockMailmanServer from .MockCloudStackServer import MockCloudStackServer +from .MockHarborServer import MockHarborServer from .conftest_ceod_api import client # noqa: F401 from .conftest_ceo import cli_setup # noqa: F401 @@ -256,6 +258,21 @@ def mock_cloud_server(): mock_server.stop() +@pytest.fixture(scope='session') +def mock_harbor_server(): + mock_server = MockHarborServer() + mock_server.start() + yield mock_server + mock_server.stop() + + +@pytest.fixture(scope='session') +def registry_srv(cfg): + reg_srv = ContainerRegistryService() + component.getGlobalSiteManager().registerUtility(reg_srv, IContainerRegistryService) + return reg_srv + + @pytest.fixture(scope='session') def mysql_srv(cfg): mysql_srv = MySQLService() @@ -322,6 +339,7 @@ def app( cloudstack_srv, vhost_mgr, k8s_srv, + registry_srv, cloud_mgr, ): app = create_app({'TESTING': True})