Add container registry API #42
|
@ -12,6 +12,7 @@ add_fqdn_to_hosts $(get_ip_addr auth1) auth1
|
||||||
python -m tests.MockMailmanServer &
|
python -m tests.MockMailmanServer &
|
||||||
python -m tests.MockSMTPServer &
|
python -m tests.MockSMTPServer &
|
||||||
python -m tests.MockCloudStackServer &
|
python -m tests.MockCloudStackServer &
|
||||||
|
python -m tests.MockHarborServer &
|
||||||
|
|
||||||
export DEBIAN_FRONTEND=noninteractive
|
export DEBIAN_FRONTEND=noninteractive
|
||||||
apt update
|
apt update
|
||||||
|
|
|
@ -9,6 +9,7 @@ from .postgresql import postgresql
|
||||||
from .mailman import mailman
|
from .mailman import mailman
|
||||||
from .cloud import cloud
|
from .cloud import cloud
|
||||||
from .k8s import k8s
|
from .k8s import k8s
|
||||||
|
from .registry import registry
|
||||||
|
|
||||||
|
|
||||||
@click.group()
|
@click.group()
|
||||||
|
@ -25,3 +26,4 @@ cli.add_command(postgresql)
|
||||||
cli.add_command(mailman)
|
cli.add_command(mailman)
|
||||||
cli.add_command(cloud)
|
cli.add_command(cloud)
|
||||||
cli.add_command(k8s)
|
cli.add_command(k8s)
|
||||||
|
cli.add_command(registry)
|
||||||
|
|
|
@ -0,0 +1,21 @@
|
||||||
|
import click
|
||||||
|
|
||||||
|
from ..utils import http_post
|
||||||
|
from .utils import handle_sync_response
|
||||||
|
|
||||||
|
|
||||||
|
@click.group(short_help='Manage your container registry account')
|
||||||
|
def registry():
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@registry.group(short_help='Manage your container registry project')
|
||||||
|
def project():
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@project.command(short_help='Create a registry project')
|
||||||
|
def create():
|
||||||
|
resp = http_post('/api/cloud/registry/projects')
|
||||||
|
handle_sync_response(resp)
|
||||||
|
click.echo('Congratulations! Your registry project was successfully created.')
|
|
@ -0,0 +1,22 @@
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
from zope.interface import Interface
|
||||||
|
|
||||||
|
|
||||||
|
class IContainerRegistryService(Interface):
|
||||||
|
"""Manage Harbor projects and users."""
|
||||||
|
|
||||||
|
def get_accounts() -> List[str]:
|
||||||
|
"""Get a list of Harbor account usernames."""
|
||||||
|
|
||||||
|
def create_project_for_user(username: str):
|
||||||
|
"""
|
||||||
|
Create a new Harbor project for a user add make them a Project Admin.
|
||||||
|
The user needs to have logged in to Harbor at least once.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def delete_project_for_user(username: str):
|
||||||
|
"""
|
||||||
|
Deletes the Harbor project for the given user, if it exists.
|
||||||
|
All repositories in the project will be deleted.
|
||||||
|
"""
|
|
@ -4,7 +4,18 @@ from zope.interface import Interface, Attribute
|
||||||
|
|
||||||
|
|
||||||
class IUser(Interface):
|
class IUser(Interface):
|
||||||
"""Represents a Unix user."""
|
"""
|
||||||
|
Represents a Unix user.
|
||||||
|
|
||||||
|
There are four types of Unix users in the CSC LDAP:
|
||||||
|
1. Members
|
||||||
|
2. Club reps
|
||||||
|
3. Clubs
|
||||||
|
4. System accounts (e.g. syscom, sysadmin, progcom, exec, git, www)
|
||||||
|
|
||||||
|
Members can become club reps and vice versa. The last term registration
|
||||||
|
of a user determines if they are a member or club rep.
|
||||||
|
"""
|
||||||
|
|
||||||
# LDAP attributes
|
# LDAP attributes
|
||||||
uid = Attribute('user identifier')
|
uid = Attribute('user identifier')
|
||||||
|
@ -39,6 +50,14 @@ class IUser(Interface):
|
||||||
Returns False if this is the Unix user for a member.
|
Returns False if this is the Unix user for a member.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
def is_member_or_club_rep() -> bool:
|
||||||
|
"""
|
||||||
|
Returns True iff this user has the 'member' objectClass.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def is_member() -> bool:
|
||||||
|
"""Returns True iff this user is a member."""
|
||||||
|
|
||||||
def add_to_ldap():
|
def add_to_ldap():
|
||||||
"""
|
"""
|
||||||
Add a new record to LDAP for this user.
|
Add a new record to LDAP for this user.
|
||||||
|
|
|
@ -13,3 +13,4 @@ from .IHTTPClient import IHTTPClient
|
||||||
from .IDatabaseService import IDatabaseService
|
from .IDatabaseService import IDatabaseService
|
||||||
from .IVHostManager import IVHostManager
|
from .IVHostManager import IVHostManager
|
||||||
from .IKubernetesService import IKubernetesService
|
from .IKubernetesService import IKubernetesService
|
||||||
|
from .IContainerRegistryService import IContainerRegistryService
|
||||||
|
|
|
@ -8,12 +8,14 @@ from zope import component
|
||||||
from .error_handlers import register_error_handlers
|
from .error_handlers import register_error_handlers
|
||||||
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
|
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
|
||||||
IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \
|
IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \
|
||||||
ICloudStackService, ICloudResourceManager, IKubernetesService, IVHostManager
|
ICloudStackService, ICloudResourceManager, IKubernetesService, IVHostManager, \
|
||||||
|
IContainerRegistryService
|
||||||
from ceo_common.model import Config, HTTPClient, RemoteMailmanService
|
from ceo_common.model import Config, HTTPClient, RemoteMailmanService
|
||||||
from ceod.api.spnego import init_spnego
|
from ceod.api.spnego import init_spnego
|
||||||
from ceod.model import KerberosService, LDAPService, FileService, \
|
from ceod.model import KerberosService, LDAPService, FileService, \
|
||||||
MailmanService, MailService, UWLDAPService, CloudStackService, \
|
MailmanService, MailService, UWLDAPService, CloudStackService, \
|
||||||
CloudResourceManager, KubernetesService, VHostManager
|
CloudResourceManager, KubernetesService, VHostManager, \
|
||||||
|
ContainerRegistryService
|
||||||
from ceod.db import MySQLService, PostgreSQLService
|
from ceod.db import MySQLService, PostgreSQLService
|
||||||
|
|
||||||
|
|
||||||
|
@ -123,7 +125,7 @@ def register_services(app):
|
||||||
psql_srv = PostgreSQLService()
|
psql_srv = PostgreSQLService()
|
||||||
component.provideUtility(psql_srv, IDatabaseService, 'postgresql')
|
component.provideUtility(psql_srv, IDatabaseService, 'postgresql')
|
||||||
|
|
||||||
# CloudStackService, CloudResourceManager, VHostManager, KubernetesService
|
# all of the cloud services
|
||||||
if hostname == cfg.get('ceod_cloud_host'):
|
if hostname == cfg.get('ceod_cloud_host'):
|
||||||
cloudstack_srv = CloudStackService()
|
cloudstack_srv = CloudStackService()
|
||||||
component.provideUtility(cloudstack_srv, ICloudStackService)
|
component.provideUtility(cloudstack_srv, ICloudStackService)
|
||||||
|
@ -136,3 +138,6 @@ def register_services(app):
|
||||||
|
|
||||||
k8s_srv = KubernetesService()
|
k8s_srv = KubernetesService()
|
||||||
component.provideUtility(k8s_srv, IKubernetesService)
|
component.provideUtility(k8s_srv, IKubernetesService)
|
||||||
|
|
||||||
|
reg_srv = ContainerRegistryService()
|
||||||
|
component.provideUtility(reg_srv, IContainerRegistryService)
|
||||||
|
|
|
@ -4,7 +4,7 @@ from zope import component
|
||||||
from .utils import requires_authentication_no_realm, authz_restrict_to_syscom, \
|
from .utils import requires_authentication_no_realm, authz_restrict_to_syscom, \
|
||||||
get_valid_member_or_throw
|
get_valid_member_or_throw
|
||||||
from ceo_common.interfaces import ICloudStackService, IVHostManager, \
|
from ceo_common.interfaces import ICloudStackService, IVHostManager, \
|
||||||
IKubernetesService, ICloudResourceManager
|
IKubernetesService, ICloudResourceManager, IContainerRegistryService
|
||||||
|
|
||||||
bp = Blueprint('cloud', __name__)
|
bp = Blueprint('cloud', __name__)
|
||||||
|
|
||||||
|
@ -62,3 +62,12 @@ def create_k8s_account(auth_user: str):
|
||||||
'status': 'OK',
|
'status': 'OK',
|
||||||
'kubeconfig': kubeconfig,
|
'kubeconfig': kubeconfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route('/registry/projects', methods=['POST'])
|
||||||
|
@requires_authentication_no_realm
|
||||||
|
def create_registry_project(auth_user: str):
|
||||||
|
get_valid_member_or_throw(auth_user)
|
||||||
|
reg_srv = component.getUtility(IContainerRegistryService)
|
||||||
|
reg_srv.create_project_for_user(auth_user)
|
||||||
|
return {'status': 'OK'}
|
||||||
|
|
|
@ -2,15 +2,16 @@ from collections import defaultdict
|
||||||
import datetime
|
import datetime
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from typing import Dict
|
from typing import Dict, List
|
||||||
|
|
||||||
from zope import component
|
from zope import component
|
||||||
from zope.interface import implementer
|
from zope.interface import implementer
|
||||||
|
|
||||||
|
from ceo_common.errors import UserNotFoundError
|
||||||
from ceo_common.logger_factory import logger_factory
|
from ceo_common.logger_factory import logger_factory
|
||||||
from ceo_common.interfaces import ICloudResourceManager, \
|
from ceo_common.interfaces import ICloudResourceManager, IUser, \
|
||||||
ILDAPService, IMailService, IKubernetesService, IVHostManager, \
|
ILDAPService, IMailService, IKubernetesService, IVHostManager, \
|
||||||
ICloudStackService
|
ICloudStackService, IContainerRegistryService
|
||||||
from ceo_common.model import Term
|
from ceo_common.model import Term
|
||||||
import ceo_common.utils as utils
|
import ceo_common.utils as utils
|
||||||
|
|
||||||
|
@ -26,79 +27,133 @@ class CloudResourceManager:
|
||||||
self.pending_deletions_file = \
|
self.pending_deletions_file = \
|
||||||
os.path.join(state_dir, 'pending_account_deletions.json')
|
os.path.join(state_dir, 'pending_account_deletions.json')
|
||||||
|
|
||||||
def purge_accounts(self) -> Dict:
|
@staticmethod
|
||||||
accounts_deleted = []
|
def _should_not_have_resources_deleted(user: IUser) -> bool:
|
||||||
accounts_to_be_deleted = []
|
return not user.is_member() or user.membership_is_valid()
|
||||||
result = {
|
|
||||||
'accounts_deleted': accounts_deleted,
|
def _get_resources_for_each_user(self) -> Dict[str, Dict]:
|
||||||
'accounts_to_be_deleted': accounts_to_be_deleted,
|
"""
|
||||||
|
Get a list of cloud resources each user is using.
|
||||||
|
The returned dict looks like
|
||||||
|
{
|
||||||
|
"ctdalek": {
|
||||||
|
"resources": ["cloudstack", "k8s", ...],
|
||||||
|
"cloudstack_account_id": "3452345-2453245-23453..."
|
||||||
|
},
|
||||||
|
...
|
||||||
}
|
}
|
||||||
|
The "cloudstack_account_id" key will only be present if the user
|
||||||
|
has a CloudStack account.
|
||||||
|
"""
|
||||||
|
k8s_srv = component.getUtility(IKubernetesService)
|
||||||
|
vhost_mgr = component.getUtility(IVHostManager)
|
||||||
|
cloudstack_srv = component.getUtility(ICloudStackService)
|
||||||
|
reg_srv = component.getUtility(IContainerRegistryService)
|
||||||
|
|
||||||
current_term = Term.current()
|
accounts = defaultdict(lambda: {'resources': []})
|
||||||
beginning_of_term = current_term.to_datetime()
|
|
||||||
now = utils.get_current_datetime()
|
|
||||||
delta = now - beginning_of_term
|
|
||||||
if delta.days < 30:
|
|
||||||
# one-month grace period
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
cloudstack_accounts = cloudstack_srv.get_accounts()
|
||||||
|
# note that cloudstack_accounts is a dict, not a list
|
||||||
|
for username, account_id in cloudstack_accounts.items():
|
||||||
|
accounts[username]['resources'].append('cloudstack')
|
||||||
|
accounts[username]['cloudstack_account_id'] = account_id
|
||||||
|
|
||||||
|
vhost_accounts = vhost_mgr.get_accounts()
|
||||||
|
for username in vhost_accounts:
|
||||||
|
accounts[username]['resources'].append('vhost')
|
||||||
|
|
||||||
|
k8s_accounts = k8s_srv.get_accounts()
|
||||||
|
for username in k8s_accounts:
|
||||||
|
accounts[username]['resources'].append('k8s')
|
||||||
|
|
||||||
|
reg_accounts = reg_srv.get_accounts()
|
||||||
|
for username in reg_accounts:
|
||||||
|
accounts[username]['resources'].append('registry')
|
||||||
|
|
||||||
|
return accounts
|
||||||
|
|
||||||
|
def _perform_deletions(
|
||||||
|
self,
|
||||||
|
state: Dict,
|
||||||
|
accounts: Dict[str, Dict],
|
||||||
|
accounts_deleted: List[str],
|
||||||
|
):
|
||||||
ldap_srv = component.getUtility(ILDAPService)
|
ldap_srv = component.getUtility(ILDAPService)
|
||||||
mail_srv = component.getUtility(IMailService)
|
mail_srv = component.getUtility(IMailService)
|
||||||
k8s_srv = component.getUtility(IKubernetesService)
|
k8s_srv = component.getUtility(IKubernetesService)
|
||||||
vhost_mgr = component.getUtility(IVHostManager)
|
vhost_mgr = component.getUtility(IVHostManager)
|
||||||
cloudstack_srv = component.getUtility(ICloudStackService)
|
cloudstack_srv = component.getUtility(ICloudStackService)
|
||||||
|
reg_srv = component.getUtility(IContainerRegistryService)
|
||||||
|
|
||||||
# get a list of all cloud services each user is using
|
for username in state['accounts_to_be_deleted']:
|
||||||
accounts = defaultdict(list)
|
if username not in accounts:
|
||||||
cloudstack_accounts = cloudstack_srv.get_accounts()
|
continue
|
||||||
# note that cloudstack_accounts is a dict, not a list
|
try:
|
||||||
for username in cloudstack_accounts:
|
|
||||||
accounts[username].append('cloudstack')
|
|
||||||
vhost_accounts = vhost_mgr.get_accounts()
|
|
||||||
for username in vhost_accounts:
|
|
||||||
accounts[username].append('vhost')
|
|
||||||
k8s_accounts = k8s_srv.get_accounts()
|
|
||||||
for username in k8s_accounts:
|
|
||||||
accounts[username].append('k8s')
|
|
||||||
|
|
||||||
if os.path.isfile(self.pending_deletions_file):
|
|
||||||
state = json.load(open(self.pending_deletions_file))
|
|
||||||
last_check = datetime.datetime.fromtimestamp(state['timestamp'])
|
|
||||||
delta = now - last_check
|
|
||||||
if delta.days < 7:
|
|
||||||
logger.debug(
|
|
||||||
'Skipping account purge because less than one week has '
|
|
||||||
'passed since the warning emails were sent out'
|
|
||||||
)
|
|
||||||
accounts_to_be_deleted.extend(state['accounts_to_be_deleted'])
|
|
||||||
return result
|
|
||||||
for username in state['accounts_to_be_deleted']:
|
|
||||||
if username not in accounts:
|
|
||||||
continue
|
|
||||||
user = ldap_srv.get_user(username)
|
user = ldap_srv.get_user(username)
|
||||||
if user.membership_is_valid():
|
except UserNotFoundError:
|
||||||
continue
|
continue
|
||||||
services = accounts[username]
|
if self._should_not_have_resources_deleted(user):
|
||||||
if 'cloudstack' in services:
|
continue
|
||||||
account_id = cloudstack_accounts[username]
|
resources = accounts[username]['resources']
|
||||||
cloudstack_srv.delete_account(account_id)
|
if 'cloudstack' in resources:
|
||||||
if 'vhost' in services:
|
account_id = accounts[username]['cloudstack_account_id']
|
||||||
vhost_mgr.delete_all_vhosts_for_user(username)
|
cloudstack_srv.delete_account(account_id)
|
||||||
if 'k8s' in services:
|
if 'vhost' in resources:
|
||||||
k8s_srv.delete_account(username)
|
vhost_mgr.delete_all_vhosts_for_user(username)
|
||||||
accounts_deleted.append(username)
|
if 'k8s' in resources:
|
||||||
mail_srv.send_cloud_account_has_been_deleted_message(user)
|
k8s_srv.delete_account(username)
|
||||||
logger.info(f'Deleted cloud resources for {username}')
|
if 'registry' in resources:
|
||||||
os.unlink(self.pending_deletions_file)
|
reg_srv.delete_project_for_user(username)
|
||||||
return result
|
accounts_deleted.append(username)
|
||||||
|
mail_srv.send_cloud_account_has_been_deleted_message(user)
|
||||||
|
logger.info(f'Deleted cloud resources for {username}')
|
||||||
|
|
||||||
|
def _perform_deletions_if_warning_period_passed(
|
||||||
|
self,
|
||||||
|
now: datetime.datetime,
|
||||||
|
accounts: Dict[str, Dict],
|
||||||
|
accounts_deleted: List[str],
|
||||||
|
accounts_to_be_deleted: List[str],
|
||||||
|
):
|
||||||
|
state = json.load(open(self.pending_deletions_file))
|
||||||
|
last_check = datetime.datetime.fromtimestamp(state['timestamp'])
|
||||||
|
delta = now - last_check
|
||||||
|
if delta.days < 7:
|
||||||
|
logger.debug(
|
||||||
|
'Skipping account purge because less than one week has '
|
||||||
|
'passed since the warning emails were sent out'
|
||||||
|
)
|
||||||
|
accounts_to_be_deleted.extend(state['accounts_to_be_deleted'])
|
||||||
|
return
|
||||||
|
self._perform_deletions(state, accounts, accounts_deleted)
|
||||||
|
os.unlink(self.pending_deletions_file)
|
||||||
|
|
||||||
|
def _in_grace_period(self, now: datetime.datetime) -> bool:
|
||||||
|
current_term = Term.current()
|
||||||
|
beginning_of_term = current_term.to_datetime()
|
||||||
|
delta = now - beginning_of_term
|
||||||
|
# one-month grace period
|
||||||
|
return delta.days < 30
|
||||||
|
|
||||||
|
def _send_out_warning_emails(
|
||||||
|
self,
|
||||||
|
now: datetime.datetime,
|
||||||
|
accounts: Dict[str, dict],
|
||||||
|
accounts_to_be_deleted: List[str],
|
||||||
|
):
|
||||||
|
ldap_srv = component.getUtility(ILDAPService)
|
||||||
|
mail_srv = component.getUtility(IMailService)
|
||||||
state = {
|
state = {
|
||||||
'timestamp': int(now.timestamp()),
|
'timestamp': int(now.timestamp()),
|
||||||
'accounts_to_be_deleted': accounts_to_be_deleted,
|
'accounts_to_be_deleted': accounts_to_be_deleted,
|
||||||
}
|
}
|
||||||
for username in accounts:
|
for username in accounts:
|
||||||
user = ldap_srv.get_user(username)
|
try:
|
||||||
if user.membership_is_valid():
|
user = ldap_srv.get_user(username)
|
||||||
|
except UserNotFoundError:
|
||||||
|
logger.warning(f'User {username} not found')
|
||||||
|
continue
|
||||||
|
if self._should_not_have_resources_deleted(user):
|
||||||
continue
|
continue
|
||||||
accounts_to_be_deleted.append(username)
|
accounts_to_be_deleted.append(username)
|
||||||
mail_srv.send_cloud_account_will_be_deleted_message(user)
|
mail_srv.send_cloud_account_will_be_deleted_message(user)
|
||||||
|
@ -108,4 +163,30 @@ class CloudResourceManager:
|
||||||
)
|
)
|
||||||
if accounts_to_be_deleted:
|
if accounts_to_be_deleted:
|
||||||
json.dump(state, open(self.pending_deletions_file, 'w'))
|
json.dump(state, open(self.pending_deletions_file, 'w'))
|
||||||
|
|
||||||
|
def _warning_emails_were_sent_out(self) -> bool:
|
||||||
|
return os.path.isfile(self.pending_deletions_file)
|
||||||
|
|
||||||
|
def purge_accounts(self) -> Dict:
|
||||||
|
accounts_deleted = []
|
||||||
|
accounts_to_be_deleted = []
|
||||||
|
result = {
|
||||||
|
'accounts_deleted': accounts_deleted,
|
||||||
|
'accounts_to_be_deleted': accounts_to_be_deleted,
|
||||||
|
}
|
||||||
|
|
||||||
|
now = utils.get_current_datetime()
|
||||||
|
if self._in_grace_period(now):
|
||||||
|
return result
|
||||||
|
|
||||||
|
# get a list of all cloud services each user is using
|
||||||
|
accounts = self._get_resources_for_each_user()
|
||||||
|
|
||||||
|
if self._warning_emails_were_sent_out():
|
||||||
|
self._perform_deletions_if_warning_period_passed(
|
||||||
|
now, accounts, accounts_deleted, accounts_to_be_deleted)
|
||||||
|
return result
|
||||||
|
|
||||||
|
self._send_out_warning_emails(now, accounts, accounts_to_be_deleted)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
|
@ -0,0 +1,87 @@
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from requests.auth import HTTPBasicAuth
|
||||||
|
from zope import component
|
||||||
|
from zope.interface import implementer
|
||||||
|
|
||||||
|
from ceo_common.errors import UserNotFoundError
|
||||||
|
from ceo_common.interfaces import IContainerRegistryService, IConfig
|
||||||
|
|
||||||
|
|
||||||
|
@implementer(IContainerRegistryService)
|
||||||
|
class ContainerRegistryService:
|
||||||
|
def __init__(self):
|
||||||
|
cfg = component.getUtility(IConfig)
|
||||||
|
self.base_url = cfg.get('registry_base_url')
|
||||||
|
if self.base_url.endswith('/'):
|
||||||
|
self.base_url = self.base_url[:-1]
|
||||||
|
api_username = cfg.get('registry_username')
|
||||||
|
api_password = cfg.get('registry_password')
|
||||||
|
self.basic_auth = HTTPBasicAuth(api_username, api_password)
|
||||||
|
|
||||||
|
def _http_request(self, method: str, path: str, **kwargs):
|
||||||
|
return requests.request(
|
||||||
|
method, self.base_url + path, **kwargs, auth=self.basic_auth)
|
||||||
|
|
||||||
|
def _http_get(self, path: str, **kwargs):
|
||||||
|
return self._http_request('GET', path, **kwargs)
|
||||||
|
|
||||||
|
def _http_post(self, path, **kwargs):
|
||||||
|
return self._http_request('POST', path, **kwargs)
|
||||||
|
|
||||||
|
def _http_delete(self, path, **kwargs):
|
||||||
|
return self._http_request('DELETE', path, **kwargs)
|
||||||
|
|
||||||
|
def _get_account(self, username: str):
|
||||||
|
resp = self._http_get('/users', params={'username': username})
|
||||||
|
users = resp.json()
|
||||||
|
if len(users) < 1:
|
||||||
|
raise UserNotFoundError(username)
|
||||||
|
return users[0]
|
||||||
|
|
||||||
|
def get_accounts(self) -> List[str]:
|
||||||
|
# We're only interested in accounts which have a project, so
|
||||||
|
# we're actually just going to get a list of projects
|
||||||
|
resp = self._http_get('/projects')
|
||||||
|
resp.raise_for_status()
|
||||||
|
# This project is only owned by the admin account, so we don't
|
||||||
|
# want to include it
|
||||||
|
project_exceptions = ['library']
|
||||||
|
return [
|
||||||
|
project['name'] for project in resp.json()
|
||||||
|
if project['name'] not in project_exceptions
|
||||||
|
]
|
||||||
|
|
||||||
|
def create_project_for_user(self, username: str):
|
||||||
|
user_id = self._get_account(username)['user_id']
|
||||||
|
|
||||||
|
# Create the project
|
||||||
|
resp = self._http_post(
|
||||||
|
'/projects', json={'project_name': username, 'public': True})
|
||||||
|
# 409 => project already exists (that is OK)
|
||||||
|
if resp.status_code != 409:
|
||||||
|
resp.raise_for_status()
|
||||||
|
|
||||||
|
# Add the user as a project admin (role ID 1)
|
||||||
|
resp = self._http_post(
|
||||||
|
f'/projects/{username}/members',
|
||||||
|
json={'role_id': 1, 'member_user': {'user_id': user_id}})
|
||||||
|
# 409 => project member already exists (that is OK)
|
||||||
|
if resp.status_code != 409:
|
||||||
|
resp.raise_for_status()
|
||||||
|
|
||||||
|
def delete_project_for_user(self, username: str):
|
||||||
|
# Delete all of the repositories inside the project first
|
||||||
|
resp = self._http_get(f'/projects/{username}/repositories')
|
||||||
|
if resp.status_code == 403:
|
||||||
|
# For some reason a 403 is returned if the project doesn't exist
|
||||||
|
return
|
||||||
|
resp.raise_for_status()
|
||||||
|
repositories = [repo['name'] for repo in resp.json()]
|
||||||
|
for repo in repositories:
|
||||||
|
resp = self._http_delete(f'/projects/{username}/repositories/{repo}')
|
||||||
|
resp.raise_for_status()
|
||||||
|
# Delete the project now that it is empty
|
||||||
|
resp = self._http_delete(f'/projects/{username}')
|
||||||
|
resp.raise_for_status()
|
|
@ -32,6 +32,7 @@ class User:
|
||||||
mail_local_addresses: Union[List[str], None] = None,
|
mail_local_addresses: Union[List[str], None] = None,
|
||||||
is_club_rep: Union[bool, None] = None,
|
is_club_rep: Union[bool, None] = None,
|
||||||
is_club: bool = False,
|
is_club: bool = False,
|
||||||
|
is_member_or_club_rep: Union[bool, None] = None,
|
||||||
ldap3_entry: Union[ldap3.Entry, None] = None,
|
ldap3_entry: Union[ldap3.Entry, None] = None,
|
||||||
shadowExpire: Union[int, None] = None,
|
shadowExpire: Union[int, None] = None,
|
||||||
):
|
):
|
||||||
|
@ -61,11 +62,13 @@ class User:
|
||||||
if is_club_rep is None:
|
if is_club_rep is None:
|
||||||
if is_club:
|
if is_club:
|
||||||
# not a real user
|
# not a real user
|
||||||
self.is_club_rep = False
|
is_club_rep = False
|
||||||
else:
|
else:
|
||||||
self.is_club_rep = should_be_club_rep(terms, non_member_terms)
|
is_club_rep = should_be_club_rep(terms, non_member_terms)
|
||||||
else:
|
self.is_club_rep = is_club_rep
|
||||||
self.is_club_rep = is_club_rep
|
if is_member_or_club_rep is None:
|
||||||
|
is_member_or_club_rep = terms is not None or non_member_terms is not None
|
||||||
|
self._is_member_or_club_rep = is_member_or_club_rep
|
||||||
self.ldap3_entry = ldap3_entry
|
self.ldap3_entry = ldap3_entry
|
||||||
self.shadowExpire = shadowExpire
|
self.shadowExpire = shadowExpire
|
||||||
|
|
||||||
|
@ -107,6 +110,12 @@ class User:
|
||||||
def is_club(self) -> bool:
|
def is_club(self) -> bool:
|
||||||
return self._is_club
|
return self._is_club
|
||||||
|
|
||||||
|
def is_member_or_club_rep(self) -> bool:
|
||||||
|
return self._is_member_or_club_rep
|
||||||
|
|
||||||
|
def is_member(self):
|
||||||
|
return self.is_member_or_club_rep() and not self.is_club_rep
|
||||||
|
|
||||||
def add_to_ldap(self):
|
def add_to_ldap(self):
|
||||||
if not self.mail_local_addresses:
|
if not self.mail_local_addresses:
|
||||||
self.mail_local_addresses = [f'{self.uid}@{self.base_domain}']
|
self.mail_local_addresses = [f'{self.uid}@{self.base_domain}']
|
||||||
|
@ -158,6 +167,7 @@ class User:
|
||||||
mail_local_addresses=attrs.get('mailLocalAddress'),
|
mail_local_addresses=attrs.get('mailLocalAddress'),
|
||||||
is_club_rep=attrs.get('isClubRep', [False])[0],
|
is_club_rep=attrs.get('isClubRep', [False])[0],
|
||||||
is_club=('club' in attrs['objectClass']),
|
is_club=('club' in attrs['objectClass']),
|
||||||
|
is_member_or_club_rep=('member' in attrs['objectClass']),
|
||||||
shadowExpire=attrs.get('shadowExpire'),
|
shadowExpire=attrs.get('shadowExpire'),
|
||||||
ldap3_entry=entry,
|
ldap3_entry=entry,
|
||||||
)
|
)
|
||||||
|
|
|
@ -12,3 +12,4 @@ from .MailService import MailService
|
||||||
from .MailmanService import MailmanService
|
from .MailmanService import MailmanService
|
||||||
from .VHostManager import VHostManager
|
from .VHostManager import VHostManager
|
||||||
from .KubernetesService import KubernetesService
|
from .KubernetesService import KubernetesService
|
||||||
|
from .ContainerRegistryService import ContainerRegistryService
|
||||||
|
|
|
@ -100,3 +100,8 @@ members_clusterrole = csc-members-default
|
||||||
members_group = csc-members
|
members_group = csc-members
|
||||||
authority_cert_path = /etc/csc/k8s-authority.crt
|
authority_cert_path = /etc/csc/k8s-authority.crt
|
||||||
server_url = https://172.19.134.149:6443
|
server_url = https://172.19.134.149:6443
|
||||||
|
|
||||||
|
[registry]
|
||||||
|
base_url = https://registry.cloud.csclub.uwaterloo.ca/api/v2.0
|
||||||
|
username = REPLACE_ME
|
||||||
|
password = REPLACE_ME
|
||||||
|
|
|
@ -0,0 +1,104 @@
|
||||||
|
from aiohttp import web
|
||||||
|
|
||||||
|
from .MockHTTPServerBase import MockHTTPServerBase
|
||||||
|
|
||||||
|
|
||||||
|
class MockHarborServer(MockHTTPServerBase):
|
||||||
|
def __init__(self, port=8002):
|
||||||
|
prefix = '/api/v2.0'
|
||||||
|
routes = [
|
||||||
|
web.get(prefix + '/users', self.users_get_handler),
|
||||||
|
web.get(prefix + '/projects', self.projects_get_handler),
|
||||||
|
web.post(prefix + '/projects', self.projects_post_handler),
|
||||||
|
web.post(prefix + '/projects/{project}/members', self.members_post_handler),
|
||||||
|
web.get(prefix + '/projects/{project}/repositories', self.repositories_get_handler),
|
||||||
|
web.delete(prefix + '/projects/{project}/repositories/{repository}', self.repositories_delete_handler),
|
||||||
|
web.delete(prefix + '/projects/{project}', self.projects_delete_handler),
|
||||||
|
# for debugging purposes
|
||||||
|
web.post('/reset', self.reset_handler),
|
||||||
|
web.delete('/users/{username}', self.users_delete_handler),
|
||||||
|
]
|
||||||
|
super().__init__(port, routes)
|
||||||
|
|
||||||
|
self.users = ['ctdalek', 'regular1', 'exec1']
|
||||||
|
self.projects = {
|
||||||
|
'ctdalek': ['repo1', 'repo2'],
|
||||||
|
'regular1': [],
|
||||||
|
'exec1': [],
|
||||||
|
}
|
||||||
|
|
||||||
|
async def projects_get_handler(self, request):
|
||||||
|
return web.json_response([
|
||||||
|
{'name': name, 'project_id': i + 1}
|
||||||
|
for i, name in enumerate(self.projects.keys())
|
||||||
|
])
|
||||||
|
|
||||||
|
async def projects_delete_handler(self, request):
|
||||||
|
project_name = request.match_info['project']
|
||||||
|
if project_name not in self.projects:
|
||||||
|
return web.json_response({"errors": [{
|
||||||
|
"code": "FORBIDDEN", "message": "forbidden"
|
||||||
|
}]}, status=403)
|
||||||
|
del self.projects[project_name]
|
||||||
|
return web.Response(text='', status=200)
|
||||||
|
|
||||||
|
async def repositories_delete_handler(self, request):
|
||||||
|
project_name = request.match_info['project']
|
||||||
|
repository_name = request.match_info['repository']
|
||||||
|
self.projects[project_name].remove(repository_name)
|
||||||
|
return web.Response(text='', status=200)
|
||||||
|
|
||||||
|
async def repositories_get_handler(self, request):
|
||||||
|
project_name = request.match_info['project']
|
||||||
|
if project_name not in self.projects:
|
||||||
|
return web.json_response({"errors": [{
|
||||||
|
"code": "FORBIDDEN", "message": "forbidden"
|
||||||
|
}]}, status=403)
|
||||||
|
projects = self.projects[project_name]
|
||||||
|
return web.json_response([
|
||||||
|
{'id': i, 'name': name} for i, name in enumerate(projects)
|
||||||
|
])
|
||||||
|
|
||||||
|
async def users_get_handler(self, request):
|
||||||
|
username = request.query['username']
|
||||||
|
if username not in self.users:
|
||||||
|
return web.json_response([])
|
||||||
|
return web.json_response([{
|
||||||
|
'username': username,
|
||||||
|
'realname': username,
|
||||||
|
'user_id': self.users.index(username),
|
||||||
|
'email': username + '@csclub.internal',
|
||||||
|
}])
|
||||||
|
|
||||||
|
async def members_post_handler(self, request):
|
||||||
|
await request.json()
|
||||||
|
return web.Response(text='', status=201)
|
||||||
|
|
||||||
|
async def projects_post_handler(self, request):
|
||||||
|
body = await request.json()
|
||||||
|
project_name = body['project_name']
|
||||||
|
if project_name in self.projects:
|
||||||
|
return web.json_response({'errors': [{
|
||||||
|
"code": "CONFLICT",
|
||||||
|
"message": f"The project named {project_name} already exists",
|
||||||
|
}]}, status=409)
|
||||||
|
self.projects[project_name] = ['repo1', 'repo2']
|
||||||
|
return web.Response(text='', status=201)
|
||||||
|
|
||||||
|
async def users_delete_handler(self, request):
|
||||||
|
username = request.match_info['username']
|
||||||
|
self.users.remove(username)
|
||||||
|
return web.Response(text='OK\n', status=201)
|
||||||
|
|
||||||
|
def reset(self):
|
||||||
|
self.users.clear()
|
||||||
|
self.projects.clear()
|
||||||
|
|
||||||
|
async def reset_handler(self, request):
|
||||||
|
self.reset()
|
||||||
|
return web.Response(text='OK\n')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
server = MockHarborServer()
|
||||||
|
server.start()
|
|
@ -76,3 +76,15 @@ def test_k8s_account_activate(cli_setup, new_user):
|
||||||
assert result.exit_code == 0
|
assert result.exit_code == 0
|
||||||
assert result.output == expected
|
assert result.output == expected
|
||||||
assert os.path.isfile(os.path.join(new_user.home_directory, '.kube', 'config'))
|
assert os.path.isfile(os.path.join(new_user.home_directory, '.kube', 'config'))
|
||||||
|
|
||||||
|
|
||||||
|
def test_registry_project_create(cli_setup, mock_harbor_server, new_user):
|
||||||
|
uid = new_user.uid
|
||||||
|
runner = CliRunner()
|
||||||
|
mock_harbor_server.reset()
|
||||||
|
mock_harbor_server.users.append(uid)
|
||||||
|
with gssapi_token_ctx(uid):
|
||||||
|
result = runner.invoke(cli, ['registry', 'project', 'create'])
|
||||||
|
expected = 'Congratulations! Your registry project was successfully created.\n'
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert result.output == expected
|
||||||
|
|
|
@ -35,8 +35,8 @@ def test_create_account(client, mock_cloud_server, new_user, ldap_conn):
|
||||||
|
|
||||||
|
|
||||||
def test_purge_accounts(
|
def test_purge_accounts(
|
||||||
client, mock_cloud_server, cloud_mgr, mock_mail_server, new_user,
|
client, mock_cloud_server, cloud_mgr, mock_mail_server,
|
||||||
ldap_conn,
|
mock_harbor_server, new_user, ldap_conn,
|
||||||
):
|
):
|
||||||
uid = new_user.uid
|
uid = new_user.uid
|
||||||
mock_cloud_server.clear()
|
mock_cloud_server.clear()
|
||||||
|
@ -167,12 +167,14 @@ def test_cloud_vhosts(cfg, client, new_user, ldap_conn):
|
||||||
assert status == 403
|
assert status == 403
|
||||||
|
|
||||||
|
|
||||||
def test_cloud_vhosts_purged_account(
|
def test_cloud_resources_purged_account(
|
||||||
cfg, client, mock_cloud_server, mock_mail_server, new_user, ldap_conn,
|
cfg, client, mock_cloud_server, mock_mail_server, mock_harbor_server,
|
||||||
|
new_user, ldap_conn,
|
||||||
):
|
):
|
||||||
uid = new_user.uid
|
uid = new_user.uid
|
||||||
members_domain = cfg.get('cloud vhosts_members_domain')
|
members_domain = cfg.get('cloud vhosts_members_domain')
|
||||||
mock_cloud_server.clear()
|
mock_cloud_server.clear()
|
||||||
|
mock_harbor_server.reset()
|
||||||
current_term = Term.current()
|
current_term = Term.current()
|
||||||
beginning_of_term = current_term.to_datetime()
|
beginning_of_term = current_term.to_datetime()
|
||||||
domain1 = uid + '.' + members_domain
|
domain1 = uid + '.' + members_domain
|
||||||
|
@ -182,6 +184,9 @@ def test_cloud_vhosts_purged_account(
|
||||||
client.put(
|
client.put(
|
||||||
f'/api/cloud/vhosts/{domain1}', json={'ip_address': ip1},
|
f'/api/cloud/vhosts/{domain1}', json={'ip_address': ip1},
|
||||||
principal=uid)
|
principal=uid)
|
||||||
|
mock_harbor_server.users.append(uid)
|
||||||
|
client.post('/api/cloud/registry/projects', principal=uid)
|
||||||
|
assert len(mock_harbor_server.projects) == 1
|
||||||
|
|
||||||
expire_member(new_user, ldap_conn)
|
expire_member(new_user, ldap_conn)
|
||||||
with patch.object(ceo_common_utils, 'get_current_datetime') as now_mock:
|
with patch.object(ceo_common_utils, 'get_current_datetime') as now_mock:
|
||||||
|
@ -198,6 +203,9 @@ def test_cloud_vhosts_purged_account(
|
||||||
assert status == 200
|
assert status == 200
|
||||||
assert data == {'vhosts': []}
|
assert data == {'vhosts': []}
|
||||||
|
|
||||||
|
# registry project should have been deleted
|
||||||
|
assert len(mock_harbor_server.projects) == 0
|
||||||
|
|
||||||
mock_mail_server.messages.clear()
|
mock_mail_server.messages.clear()
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,24 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from ceo_common.errors import UserNotFoundError
|
||||||
|
|
||||||
|
|
||||||
|
def test_registry(mock_harbor_server, registry_srv):
|
||||||
|
mock_harbor_server.reset()
|
||||||
|
username = 'test1'
|
||||||
|
|
||||||
|
with pytest.raises(UserNotFoundError):
|
||||||
|
registry_srv.create_project_for_user(username)
|
||||||
|
|
||||||
|
mock_harbor_server.users.append(username)
|
||||||
|
registry_srv.create_project_for_user(username)
|
||||||
|
assert username in mock_harbor_server.projects
|
||||||
|
# trying to create a project with the same name should have no effect
|
||||||
|
registry_srv.create_project_for_user(username)
|
||||||
|
|
||||||
|
assert registry_srv.get_accounts() == [username]
|
||||||
|
|
||||||
|
registry_srv.delete_project_for_user(username)
|
||||||
|
assert username not in mock_harbor_server.projects
|
||||||
|
# trying to delete a nonexistent project should have no effect
|
||||||
|
registry_srv.delete_project_for_user(username)
|
|
@ -94,3 +94,8 @@ members_clusterrole = csc-members-default
|
||||||
members_group = csc-members
|
members_group = csc-members
|
||||||
authority_cert_path = /etc/csc/k8s-authority.crt
|
authority_cert_path = /etc/csc/k8s-authority.crt
|
||||||
server_url = https://172.19.134.149:6443
|
server_url = https://172.19.134.149:6443
|
||||||
|
|
||||||
|
[registry]
|
||||||
|
base_url = http://localhost:8002/api/v2.0
|
||||||
|
username = REPLACE_ME
|
||||||
|
password = REPLACE_ME
|
||||||
|
|
|
@ -93,3 +93,8 @@ members_clusterrole = csc-members-default
|
||||||
members_group = csc-members
|
members_group = csc-members
|
||||||
authority_cert_path = /etc/csc/k8s-authority.crt
|
authority_cert_path = /etc/csc/k8s-authority.crt
|
||||||
server_url = https://172.19.134.149:6443
|
server_url = https://172.19.134.149:6443
|
||||||
|
|
||||||
|
[registry]
|
||||||
|
base_url = http://localhost:8002/api/v2.0
|
||||||
|
username = REPLACE_ME
|
||||||
|
password = REPLACE_ME
|
||||||
|
|
|
@ -26,16 +26,18 @@ from .utils import ( # noqa: F401
|
||||||
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
|
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
|
||||||
IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
|
IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
|
||||||
IDatabaseService, ICloudStackService, IKubernetesService, IVHostManager, \
|
IDatabaseService, ICloudStackService, IKubernetesService, IVHostManager, \
|
||||||
ICloudResourceManager
|
ICloudResourceManager, IContainerRegistryService
|
||||||
from ceo_common.model import Config, HTTPClient, Term
|
from ceo_common.model import Config, HTTPClient, Term
|
||||||
from ceod.api import create_app
|
from ceod.api import create_app
|
||||||
from ceod.db import MySQLService, PostgreSQLService
|
from ceod.db import MySQLService, PostgreSQLService
|
||||||
from ceod.model import KerberosService, LDAPService, FileService, User, \
|
from ceod.model import KerberosService, LDAPService, FileService, User, \
|
||||||
MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \
|
MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \
|
||||||
CloudStackService, KubernetesService, VHostManager, CloudResourceManager
|
CloudStackService, KubernetesService, VHostManager, CloudResourceManager, \
|
||||||
|
ContainerRegistryService
|
||||||
from .MockSMTPServer import MockSMTPServer
|
from .MockSMTPServer import MockSMTPServer
|
||||||
from .MockMailmanServer import MockMailmanServer
|
from .MockMailmanServer import MockMailmanServer
|
||||||
from .MockCloudStackServer import MockCloudStackServer
|
from .MockCloudStackServer import MockCloudStackServer
|
||||||
|
from .MockHarborServer import MockHarborServer
|
||||||
from .conftest_ceod_api import client # noqa: F401
|
from .conftest_ceod_api import client # noqa: F401
|
||||||
from .conftest_ceo import cli_setup # noqa: F401
|
from .conftest_ceo import cli_setup # noqa: F401
|
||||||
|
|
||||||
|
@ -256,6 +258,21 @@ def mock_cloud_server():
|
||||||
mock_server.stop()
|
mock_server.stop()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='session')
|
||||||
|
def mock_harbor_server():
|
||||||
|
mock_server = MockHarborServer()
|
||||||
|
mock_server.start()
|
||||||
|
yield mock_server
|
||||||
|
mock_server.stop()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='session')
|
||||||
|
def registry_srv(cfg):
|
||||||
|
reg_srv = ContainerRegistryService()
|
||||||
|
component.getGlobalSiteManager().registerUtility(reg_srv, IContainerRegistryService)
|
||||||
|
return reg_srv
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope='session')
|
@pytest.fixture(scope='session')
|
||||||
def mysql_srv(cfg):
|
def mysql_srv(cfg):
|
||||||
mysql_srv = MySQLService()
|
mysql_srv = MySQLService()
|
||||||
|
@ -322,6 +339,7 @@ def app(
|
||||||
cloudstack_srv,
|
cloudstack_srv,
|
||||||
vhost_mgr,
|
vhost_mgr,
|
||||||
k8s_srv,
|
k8s_srv,
|
||||||
|
registry_srv,
|
||||||
cloud_mgr,
|
cloud_mgr,
|
||||||
):
|
):
|
||||||
app = create_app({'TESTING': True})
|
app = create_app({'TESTING': True})
|
||||||
|
|
Loading…
Reference in New Issue