use CloudResourceManager
This commit is contained in:
parent
e14b261805
commit
470b442e4c
|
@ -7,14 +7,12 @@ elif [ "$1" = delete ]; then
|
||||||
elif [ "$1" = certificate ]; then
|
elif [ "$1" = certificate ]; then
|
||||||
exit
|
exit
|
||||||
elif [ "$1" = get ]; then
|
elif [ "$1" = get ]; then
|
||||||
if [ "$2" = csr -a "$4" = "-o" ]; then
|
if [ "$2" = csr -a "$4" = "-o" -a "$5" = 'jsonpath={.status.certificate}' ]; then
|
||||||
if [ "$5" = 'jsonpath={.status.conditions[0].type}' ]; then
|
echo -n 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMxekNDQWI4Q0ZHeVk0ZVpVMnAvTjMzU0pCTlptMm1vSlE5TXFNQTBHQ1NxR1NJYjNEUUVCQ3dVQU1DZ3gKRURBT0JnTlZCQU1NQjJOMFpHRnNaV3N4RkRBU0JnTlZCQW9NQzJOell5MXRaVzFpWlhKek1CNFhEVEl4TVRJeApNekEwTWpJek4xb1hEVEl5TURFeE1qQTBNakl6TjFvd0tERVFNQTRHQTFVRUF3d0hZM1JrWVd4bGF6RVVNQklHCkExVUVDZ3dMWTNOakxXMWxiV0psY25Nd2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUIKQVFEV09vaTd6ejE0c3VBZ0V2QkgrSHFHSzlCUUlQTm5QQ0llVkxXenlFRTNxUWZRV2YvcWNzeGNST2pSKzVCTgpKSXBaQlNZdjRmNE52WFZqaHlQendoWUd0bXJRYksyT3RCTDlqMDJMWjhMVHp2TnE0MW9CYVdXUFhhaVdIVys2CjkzQnlBdXFPMmdnSEt0elNkV09TcTZpeFBXMVNGUzJRMkFWaXdZUEg3b1pQYnZacUZvMzdhbVdwd1pWUHVuVi8KV2tFRUttNUVqV05DSVUzVWpPdS9HeEJOT1g0WEpqWld4bFcwQUVROVp3K2ZSazBkdU5ScVVyUDQxbDZvcG4rKwpLRkE5NFg2NUlzcUMvMlJ4OWgrNkZFRHhIcjJPcjhOcGFuMXRjZEZHQlFyMGMxV1JxRkNHTytIM0VTeUNya1BjCmdnRDlVN3c0TmdGYkQyaVU0QXc3ZkhwakFnTUJBQUV3RFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUY3VWUwc3YKcFhSUzN1TFl1Y0k3UkRNRGpOZnFpZ0R3NnorbzZxVmxTdGpZTGpDNjFXRyswZ0g4TDJIbm5jZVYyelhjNDkrQQp6TjFna0lWT3JlRUQvRitKbGRPUGgySUpOY1pGYTBsckFFV0dNNWRRR3pDSUM0cEtmSGxOMTZ0c0w2bGdqWTYzCmUvZlhMTFdLdktDR2lRMUlBUTh4KzYyaTVvSmU3aDBlQ1Q0aEEyM0JTRnRNelo2aEdGUURNNGxxaWhHQjEyT2UKZE5yYStsNVdLemNFR21aVFBYTXNudEZVVndPejhaNld2eGo0UW1zL1dQUElKWDdLM2NiRUo4L1RQWG1tUzJrQwowNUtueUxVQzltYnR2TGZoYldhbFZVVlJVUkYwT1RaVk5mNkt6MDJWYlRqQjRJQXdyWGZKZC9lMkMvNFpGWlJTCjVWMnlJSnBJeVJGWTdQST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo='
|
||||||
echo -n Approved
|
exit
|
||||||
exit
|
elif [ "$2" = namespaces ]; then
|
||||||
elif [ "$5" = 'jsonpath={.status.certificate}' ]; then
|
echo '{"items":[{"metadata":{"name":"default"}}]}'
|
||||||
echo -n 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMxekNDQWI4Q0ZHeVk0ZVpVMnAvTjMzU0pCTlptMm1vSlE5TXFNQTBHQ1NxR1NJYjNEUUVCQ3dVQU1DZ3gKRURBT0JnTlZCQU1NQjJOMFpHRnNaV3N4RkRBU0JnTlZCQW9NQzJOell5MXRaVzFpWlhKek1CNFhEVEl4TVRJeApNekEwTWpJek4xb1hEVEl5TURFeE1qQTBNakl6TjFvd0tERVFNQTRHQTFVRUF3d0hZM1JrWVd4bGF6RVVNQklHCkExVUVDZ3dMWTNOakxXMWxiV0psY25Nd2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUIKQVFEV09vaTd6ejE0c3VBZ0V2QkgrSHFHSzlCUUlQTm5QQ0llVkxXenlFRTNxUWZRV2YvcWNzeGNST2pSKzVCTgpKSXBaQlNZdjRmNE52WFZqaHlQendoWUd0bXJRYksyT3RCTDlqMDJMWjhMVHp2TnE0MW9CYVdXUFhhaVdIVys2CjkzQnlBdXFPMmdnSEt0elNkV09TcTZpeFBXMVNGUzJRMkFWaXdZUEg3b1pQYnZacUZvMzdhbVdwd1pWUHVuVi8KV2tFRUttNUVqV05DSVUzVWpPdS9HeEJOT1g0WEpqWld4bFcwQUVROVp3K2ZSazBkdU5ScVVyUDQxbDZvcG4rKwpLRkE5NFg2NUlzcUMvMlJ4OWgrNkZFRHhIcjJPcjhOcGFuMXRjZEZHQlFyMGMxV1JxRkNHTytIM0VTeUNya1BjCmdnRDlVN3c0TmdGYkQyaVU0QXc3ZkhwakFnTUJBQUV3RFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUY3VWUwc3YKcFhSUzN1TFl1Y0k3UkRNRGpOZnFpZ0R3NnorbzZxVmxTdGpZTGpDNjFXRyswZ0g4TDJIbm5jZVYyelhjNDkrQQp6TjFna0lWT3JlRUQvRitKbGRPUGgySUpOY1pGYTBsckFFV0dNNWRRR3pDSUM0cEtmSGxOMTZ0c0w2bGdqWTYzCmUvZlhMTFdLdktDR2lRMUlBUTh4KzYyaTVvSmU3aDBlQ1Q0aEEyM0JTRnRNelo2aEdGUURNNGxxaWhHQjEyT2UKZE5yYStsNVdLemNFR21aVFBYTXNudEZVVndPejhaNld2eGo0UW1zL1dQUElKWDdLM2NiRUo4L1RQWG1tUzJrQwowNUtueUxVQzltYnR2TGZoYldhbFZVVlJVUkYwT1RaVk5mNkt6MDJWYlRqQjRJQXdyWGZKZC9lMkMvNFpGWlJTCjVWMnlJSnBJeVJGWTdQST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo='
|
exit
|
||||||
exit
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
echo 'Unrecognized command'
|
echo 'Unrecognized command'
|
||||||
|
|
|
@ -0,0 +1,16 @@
|
||||||
|
from typing import Dict
|
||||||
|
|
||||||
|
from zope.interface import Interface
|
||||||
|
|
||||||
|
|
||||||
|
class ICloudResourceManager(Interface):
|
||||||
|
"""Manages multiple cloud resources for each user."""
|
||||||
|
|
||||||
|
def purge_accounts() -> Dict:
|
||||||
|
"""
|
||||||
|
Delete cloud resources for expired CSC accounts.
|
||||||
|
A warning message will be emailed to users one week before their
|
||||||
|
cloud account is deleted.
|
||||||
|
Another message will be emailed to the users after their cloud account
|
||||||
|
has been deleted.
|
||||||
|
"""
|
|
@ -1,42 +0,0 @@
|
||||||
from typing import Dict, List
|
|
||||||
|
|
||||||
from zope.interface import Interface
|
|
||||||
|
|
||||||
from .IUser import IUser
|
|
||||||
|
|
||||||
|
|
||||||
class ICloudService(Interface):
|
|
||||||
"""Performs operations on the CSC Cloud."""
|
|
||||||
|
|
||||||
def create_account(user: IUser):
|
|
||||||
"""
|
|
||||||
Activate an LDAP account in CloudStack for the given user.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def purge_accounts() -> Dict:
|
|
||||||
"""
|
|
||||||
Delete CloudStack accounts which correspond to expired CSC accounts.
|
|
||||||
A warning message will be emailed to users one week before their
|
|
||||||
cloud account is deleted.
|
|
||||||
Another message will be emailed to the users after their cloud account
|
|
||||||
has been deleted.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def create_vhost(username: str, domain: str, ip_address: str):
|
|
||||||
"""
|
|
||||||
Create a new vhost record for the given domain and IP address.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def delete_vhost(username: str, domain: str):
|
|
||||||
"""
|
|
||||||
Delete the vhost record for the given user and domain.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def get_vhosts(username: str) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
Get the vhost records for the given user. Each record has the form
|
|
||||||
{
|
|
||||||
"domain": "app.username.m.csclub.cloud",
|
|
||||||
"ip_address": "172.19.134.12"
|
|
||||||
}
|
|
||||||
"""
|
|
|
@ -0,0 +1,26 @@
|
||||||
|
from typing import Dict
|
||||||
|
|
||||||
|
from zope.interface import Interface
|
||||||
|
|
||||||
|
from .IUser import IUser
|
||||||
|
|
||||||
|
|
||||||
|
class ICloudStackService(Interface):
|
||||||
|
"""Provides a way to interact with a CloudStack management server."""
|
||||||
|
|
||||||
|
def create_account(user: IUser):
|
||||||
|
"""
|
||||||
|
Activate an LDAP account in CloudStack for the given user.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def get_accounts() -> Dict[str, str]:
|
||||||
|
"""
|
||||||
|
Get the users who have active CloudStack accounts.
|
||||||
|
The dict is mapping of usernames to account IDs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def delete_account(account_id: str):
|
||||||
|
"""
|
||||||
|
Delete the given CloudStack account.
|
||||||
|
Note that a CloudStack account ID must be given, not a username.
|
||||||
|
"""
|
|
@ -1,3 +1,5 @@
|
||||||
|
from typing import List
|
||||||
|
|
||||||
from zope.interface import Interface
|
from zope.interface import Interface
|
||||||
|
|
||||||
|
|
||||||
|
@ -15,3 +17,8 @@ class IKubernetesService(Interface):
|
||||||
"""
|
"""
|
||||||
Delete the k8s namespace for the given user.
|
Delete the k8s namespace for the given user.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
def get_accounts() -> List[str]:
|
||||||
|
"""
|
||||||
|
Get a list of users who have k8s namespaces.
|
||||||
|
"""
|
||||||
|
|
|
@ -30,7 +30,7 @@ class IVHostManager(Interface):
|
||||||
"""
|
"""
|
||||||
Get the vhost records for the given user. Each record has the form
|
Get the vhost records for the given user. Each record has the form
|
||||||
{
|
{
|
||||||
"domain": "app.username.m.csclub.cloud",
|
"domain": "app.username.csclub.cloud",
|
||||||
"ip_address": "172.19.134.12"
|
"ip_address": "172.19.134.12"
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
|
@ -46,3 +46,8 @@ class IVHostManager(Interface):
|
||||||
Returns true iff ip_address is a valid proxy_pass target
|
Returns true iff ip_address is a valid proxy_pass target
|
||||||
in NGINX and falls within the acceptable IP range.
|
in NGINX and falls within the acceptable IP range.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
def get_accounts() -> List[str]:
|
||||||
|
"""
|
||||||
|
Get a list of users who have at least one vhost record.
|
||||||
|
"""
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
from .ICloudService import ICloudService
|
from .ICloudResourceManager import ICloudResourceManager
|
||||||
|
from .ICloudStackService import ICloudStackService
|
||||||
from .IKerberosService import IKerberosService
|
from .IKerberosService import IKerberosService
|
||||||
from .IConfig import IConfig
|
from .IConfig import IConfig
|
||||||
from .IUser import IUser
|
from .IUser import IUser
|
||||||
|
|
|
@ -8,11 +8,12 @@ from zope import component
|
||||||
from .error_handlers import register_error_handlers
|
from .error_handlers import register_error_handlers
|
||||||
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
|
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, IFileService, \
|
||||||
IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \
|
IMailmanService, IMailService, IUWLDAPService, IHTTPClient, IDatabaseService, \
|
||||||
ICloudService, IKubernetesService
|
ICloudStackService, ICloudResourceManager, IKubernetesService, IVHostManager
|
||||||
from ceo_common.model import Config, HTTPClient, RemoteMailmanService
|
from ceo_common.model import Config, HTTPClient, RemoteMailmanService
|
||||||
from ceod.api.spnego import init_spnego
|
from ceod.api.spnego import init_spnego
|
||||||
from ceod.model import KerberosService, LDAPService, FileService, \
|
from ceod.model import KerberosService, LDAPService, FileService, \
|
||||||
MailmanService, MailService, UWLDAPService, CloudService, KubernetesService
|
MailmanService, MailService, UWLDAPService, CloudStackService, \
|
||||||
|
CloudResourceManager, KubernetesService, VHostManager
|
||||||
from ceod.db import MySQLService, PostgreSQLService
|
from ceod.db import MySQLService, PostgreSQLService
|
||||||
|
|
||||||
|
|
||||||
|
@ -114,20 +115,24 @@ def register_services(app):
|
||||||
uwldap_srv = UWLDAPService()
|
uwldap_srv = UWLDAPService()
|
||||||
component.provideUtility(uwldap_srv, IUWLDAPService)
|
component.provideUtility(uwldap_srv, IUWLDAPService)
|
||||||
|
|
||||||
# MySQLService
|
# MySQLService, PostgreSQLService
|
||||||
if hostname == cfg.get('ceod_database_host'):
|
if hostname == cfg.get('ceod_database_host'):
|
||||||
mysql_srv = MySQLService()
|
mysql_srv = MySQLService()
|
||||||
component.provideUtility(mysql_srv, IDatabaseService, 'mysql')
|
component.provideUtility(mysql_srv, IDatabaseService, 'mysql')
|
||||||
|
|
||||||
# PostgreSQLService
|
|
||||||
if hostname == cfg.get('ceod_database_host'):
|
|
||||||
psql_srv = PostgreSQLService()
|
psql_srv = PostgreSQLService()
|
||||||
component.provideUtility(psql_srv, IDatabaseService, 'postgresql')
|
component.provideUtility(psql_srv, IDatabaseService, 'postgresql')
|
||||||
|
|
||||||
# CloudService, KubernetesService
|
# CloudStackService, CloudResourceManager, VHostManager, KubernetesService
|
||||||
if hostname == cfg.get('ceod_cloud_host'):
|
if hostname == cfg.get('ceod_cloud_host'):
|
||||||
cloud_srv = CloudService()
|
cloudstack_srv = CloudStackService()
|
||||||
component.provideUtility(cloud_srv, ICloudService)
|
component.provideUtility(cloudstack_srv, ICloudStackService)
|
||||||
|
|
||||||
|
cloud_mgr = CloudResourceManager()
|
||||||
|
component.provideUtility(cloud_mgr, ICloudResourceManager)
|
||||||
|
|
||||||
|
vhost_mgr = VHostManager()
|
||||||
|
component.provideUtility(vhost_mgr, IVHostManager)
|
||||||
|
|
||||||
k8s_srv = KubernetesService()
|
k8s_srv = KubernetesService()
|
||||||
component.provideUtility(k8s_srv, IKubernetesService)
|
component.provideUtility(k8s_srv, IKubernetesService)
|
||||||
|
|
|
@ -3,7 +3,8 @@ from zope import component
|
||||||
|
|
||||||
from .utils import requires_authentication_no_realm, authz_restrict_to_syscom, \
|
from .utils import requires_authentication_no_realm, authz_restrict_to_syscom, \
|
||||||
get_valid_member_or_throw
|
get_valid_member_or_throw
|
||||||
from ceo_common.interfaces import ICloudService, IKubernetesService
|
from ceo_common.interfaces import ICloudStackService, IVHostManager, \
|
||||||
|
IKubernetesService, ICloudResourceManager
|
||||||
|
|
||||||
bp = Blueprint('cloud', __name__)
|
bp = Blueprint('cloud', __name__)
|
||||||
|
|
||||||
|
@ -12,42 +13,42 @@ bp = Blueprint('cloud', __name__)
|
||||||
@requires_authentication_no_realm
|
@requires_authentication_no_realm
|
||||||
def create_account(auth_user: str):
|
def create_account(auth_user: str):
|
||||||
user = get_valid_member_or_throw(auth_user)
|
user = get_valid_member_or_throw(auth_user)
|
||||||
cloud_srv = component.getUtility(ICloudService)
|
cloudstack_srv = component.getUtility(ICloudStackService)
|
||||||
cloud_srv.create_account(user)
|
cloudstack_srv.create_account(user)
|
||||||
return {'status': 'OK'}
|
return {'status': 'OK'}
|
||||||
|
|
||||||
|
|
||||||
@bp.route('/accounts/purge', methods=['POST'])
|
@bp.route('/accounts/purge', methods=['POST'])
|
||||||
@authz_restrict_to_syscom
|
@authz_restrict_to_syscom
|
||||||
def purge_accounts():
|
def purge_accounts():
|
||||||
cloud_srv = component.getUtility(ICloudService)
|
cloud_mgr = component.getUtility(ICloudResourceManager)
|
||||||
return cloud_srv.purge_accounts()
|
return cloud_mgr.purge_accounts()
|
||||||
|
|
||||||
|
|
||||||
@bp.route('/vhosts/<domain>', methods=['PUT'])
|
@bp.route('/vhosts/<domain>', methods=['PUT'])
|
||||||
@requires_authentication_no_realm
|
@requires_authentication_no_realm
|
||||||
def create_vhost(auth_user: str, domain: str):
|
def create_vhost(auth_user: str, domain: str):
|
||||||
get_valid_member_or_throw(auth_user)
|
get_valid_member_or_throw(auth_user)
|
||||||
cloud_srv = component.getUtility(ICloudService)
|
vhost_mgr = component.getUtility(IVHostManager)
|
||||||
body = request.get_json(force=True)
|
body = request.get_json(force=True)
|
||||||
ip_address = body['ip_address']
|
ip_address = body['ip_address']
|
||||||
cloud_srv.create_vhost(auth_user, domain, ip_address)
|
vhost_mgr.create_vhost(auth_user, domain, ip_address)
|
||||||
return {'status': 'OK'}
|
return {'status': 'OK'}
|
||||||
|
|
||||||
|
|
||||||
@bp.route('/vhosts/<domain>', methods=['DELETE'])
|
@bp.route('/vhosts/<domain>', methods=['DELETE'])
|
||||||
@requires_authentication_no_realm
|
@requires_authentication_no_realm
|
||||||
def delete_vhost(auth_user: str, domain: str):
|
def delete_vhost(auth_user: str, domain: str):
|
||||||
cloud_srv = component.getUtility(ICloudService)
|
vhost_mgr = component.getUtility(IVHostManager)
|
||||||
cloud_srv.delete_vhost(auth_user, domain)
|
vhost_mgr.delete_vhost(auth_user, domain)
|
||||||
return {'status': 'OK'}
|
return {'status': 'OK'}
|
||||||
|
|
||||||
|
|
||||||
@bp.route('/vhosts', methods=['GET'])
|
@bp.route('/vhosts', methods=['GET'])
|
||||||
@requires_authentication_no_realm
|
@requires_authentication_no_realm
|
||||||
def get_vhosts(auth_user: str):
|
def get_vhosts(auth_user: str):
|
||||||
cloud_srv = component.getUtility(ICloudService)
|
vhost_mgr = component.getUtility(IVHostManager)
|
||||||
vhosts = cloud_srv.get_vhosts(auth_user)
|
vhosts = vhost_mgr.get_vhosts(auth_user)
|
||||||
return {'vhosts': vhosts}
|
return {'vhosts': vhosts}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,111 @@
|
||||||
|
from collections import defaultdict
|
||||||
|
import datetime
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from typing import Dict
|
||||||
|
|
||||||
|
from zope import component
|
||||||
|
from zope.interface import implementer
|
||||||
|
|
||||||
|
from ceo_common.logger_factory import logger_factory
|
||||||
|
from ceo_common.interfaces import ICloudResourceManager, \
|
||||||
|
ILDAPService, IMailService, IKubernetesService, IVHostManager, \
|
||||||
|
ICloudStackService
|
||||||
|
from ceo_common.model import Term
|
||||||
|
import ceo_common.utils as utils
|
||||||
|
|
||||||
|
logger = logger_factory(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@implementer(ICloudResourceManager)
|
||||||
|
class CloudResourceManager:
|
||||||
|
def __init__(self):
|
||||||
|
state_dir = '/run/ceod'
|
||||||
|
if not os.path.isdir(state_dir):
|
||||||
|
os.mkdir(state_dir)
|
||||||
|
self.pending_deletions_file = \
|
||||||
|
os.path.join(state_dir, 'pending_account_deletions.json')
|
||||||
|
|
||||||
|
def purge_accounts(self) -> Dict:
|
||||||
|
accounts_deleted = []
|
||||||
|
accounts_to_be_deleted = []
|
||||||
|
result = {
|
||||||
|
'accounts_deleted': accounts_deleted,
|
||||||
|
'accounts_to_be_deleted': accounts_to_be_deleted,
|
||||||
|
}
|
||||||
|
|
||||||
|
current_term = Term.current()
|
||||||
|
beginning_of_term = current_term.to_datetime()
|
||||||
|
now = utils.get_current_datetime()
|
||||||
|
delta = now - beginning_of_term
|
||||||
|
if delta.days < 30:
|
||||||
|
# one-month grace period
|
||||||
|
return result
|
||||||
|
|
||||||
|
ldap_srv = component.getUtility(ILDAPService)
|
||||||
|
mail_srv = component.getUtility(IMailService)
|
||||||
|
k8s_srv = component.getUtility(IKubernetesService)
|
||||||
|
vhost_mgr = component.getUtility(IVHostManager)
|
||||||
|
cloudstack_srv = component.getUtility(ICloudStackService)
|
||||||
|
|
||||||
|
# get a list of all cloud services each user is using
|
||||||
|
accounts = defaultdict(list)
|
||||||
|
cloudstack_accounts = cloudstack_srv.get_accounts()
|
||||||
|
# note that cloudstack_accounts is a dict, not a list
|
||||||
|
for username in cloudstack_accounts:
|
||||||
|
accounts[username].append('cloudstack')
|
||||||
|
vhost_accounts = vhost_mgr.get_accounts()
|
||||||
|
for username in vhost_accounts:
|
||||||
|
accounts[username].append('vhost')
|
||||||
|
k8s_accounts = k8s_srv.get_accounts()
|
||||||
|
for username in k8s_accounts:
|
||||||
|
accounts[username].append('k8s')
|
||||||
|
|
||||||
|
if os.path.isfile(self.pending_deletions_file):
|
||||||
|
state = json.load(open(self.pending_deletions_file))
|
||||||
|
last_check = datetime.datetime.fromtimestamp(state['timestamp'])
|
||||||
|
delta = now - last_check
|
||||||
|
if delta.days < 7:
|
||||||
|
logger.debug(
|
||||||
|
'Skipping account purge because less than one week has '
|
||||||
|
'passed since the warning emails were sent out'
|
||||||
|
)
|
||||||
|
accounts_to_be_deleted.extend(state['accounts_to_be_deleted'])
|
||||||
|
return result
|
||||||
|
for username in state['accounts_to_be_deleted']:
|
||||||
|
if username not in accounts:
|
||||||
|
continue
|
||||||
|
user = ldap_srv.get_user(username)
|
||||||
|
if user.membership_is_valid():
|
||||||
|
continue
|
||||||
|
services = accounts[username]
|
||||||
|
if 'cloudstack' in services:
|
||||||
|
account_id = cloudstack_accounts[username]
|
||||||
|
cloudstack_srv.delete_account(account_id)
|
||||||
|
if 'vhost' in services:
|
||||||
|
vhost_mgr.delete_all_vhosts_for_user(username)
|
||||||
|
if 'k8s' in services:
|
||||||
|
k8s_srv.delete_account(username)
|
||||||
|
accounts_deleted.append(username)
|
||||||
|
mail_srv.send_cloud_account_has_been_deleted_message(user)
|
||||||
|
logger.info(f'Deleted cloud resources for {username}')
|
||||||
|
os.unlink(self.pending_deletions_file)
|
||||||
|
return result
|
||||||
|
|
||||||
|
state = {
|
||||||
|
'timestamp': int(now.timestamp()),
|
||||||
|
'accounts_to_be_deleted': accounts_to_be_deleted,
|
||||||
|
}
|
||||||
|
for username in accounts:
|
||||||
|
user = ldap_srv.get_user(username)
|
||||||
|
if user.membership_is_valid():
|
||||||
|
continue
|
||||||
|
accounts_to_be_deleted.append(username)
|
||||||
|
mail_srv.send_cloud_account_will_be_deleted_message(user)
|
||||||
|
logger.info(
|
||||||
|
f'A warning email was sent to {username} because their '
|
||||||
|
'cloud account will be deleted'
|
||||||
|
)
|
||||||
|
if accounts_to_be_deleted:
|
||||||
|
json.dump(state, open(self.pending_deletions_file, 'w'))
|
||||||
|
return result
|
|
@ -1,232 +0,0 @@
|
||||||
from base64 import b64encode
|
|
||||||
import datetime
|
|
||||||
import hashlib
|
|
||||||
import hmac
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from typing import Dict, List
|
|
||||||
from urllib.parse import quote
|
|
||||||
|
|
||||||
import requests
|
|
||||||
from zope import component
|
|
||||||
from zope.interface import implementer
|
|
||||||
|
|
||||||
from .VHostManager import VHostManager
|
|
||||||
from ceo_common.errors import CloudStackAPIError, InvalidDomainError, \
|
|
||||||
InvalidIPError, RateLimitError
|
|
||||||
from ceo_common.logger_factory import logger_factory
|
|
||||||
from ceo_common.interfaces import ICloudService, IConfig, IUser, ILDAPService, \
|
|
||||||
IMailService, IKubernetesService
|
|
||||||
from ceo_common.model import Term
|
|
||||||
import ceo_common.utils as utils
|
|
||||||
|
|
||||||
logger = logger_factory(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
@implementer(ICloudService)
|
|
||||||
class CloudService:
|
|
||||||
def __init__(self):
|
|
||||||
cfg = component.getUtility(IConfig)
|
|
||||||
self.api_key = cfg.get('cloudstack_api_key')
|
|
||||||
self.secret_key = cfg.get('cloudstack_secret_key')
|
|
||||||
self.base_url = cfg.get('cloudstack_base_url')
|
|
||||||
self.members_domain = 'Members'
|
|
||||||
|
|
||||||
self.vhost_mgr = VHostManager()
|
|
||||||
self.max_vhosts_per_account = cfg.get('cloud vhosts_max_vhosts_per_account')
|
|
||||||
self.vhost_rate_limit_secs = cfg.get('cloud vhosts_rate_limit_seconds')
|
|
||||||
|
|
||||||
state_dir = '/run/ceod'
|
|
||||||
if not os.path.isdir(state_dir):
|
|
||||||
os.mkdir(state_dir)
|
|
||||||
self.pending_deletions_file = os.path.join(state_dir, 'cloudstack_pending_account_deletions.json')
|
|
||||||
self.vhost_rate_limit_file = os.path.join(state_dir, 'vhost_rate_limits.json')
|
|
||||||
|
|
||||||
def _create_url(self, params: Dict[str, str]) -> str:
|
|
||||||
# See https://docs.cloudstack.apache.org/en/latest/developersguide/dev.html#the-cloudstack-api
|
|
||||||
if 'apiKey' not in params and 'apikey' not in params:
|
|
||||||
params['apiKey'] = self.api_key
|
|
||||||
params['response'] = 'json'
|
|
||||||
request_str = '&'.join(
|
|
||||||
key + '=' + quote(val)
|
|
||||||
for key, val in params.items()
|
|
||||||
)
|
|
||||||
sig_str = '&'.join(
|
|
||||||
key.lower() + '=' + quote(val).lower()
|
|
||||||
for key, val in sorted(params.items())
|
|
||||||
)
|
|
||||||
sig = hmac.new(self.secret_key.encode(), sig_str.encode(), hashlib.sha1).digest()
|
|
||||||
encoded_sig = b64encode(sig).decode()
|
|
||||||
url = self.base_url + '?' + request_str + '&signature=' + quote(encoded_sig)
|
|
||||||
return url
|
|
||||||
|
|
||||||
def _get_domain_id(self, domain_name: str) -> str:
|
|
||||||
url = self._create_url({
|
|
||||||
'command': 'listDomains',
|
|
||||||
'details': 'min',
|
|
||||||
'name': domain_name,
|
|
||||||
})
|
|
||||||
resp = requests.get(url)
|
|
||||||
resp.raise_for_status()
|
|
||||||
d = resp.json()['listdomainsresponse']
|
|
||||||
assert d['count'] == 1, 'there should be one domain found'
|
|
||||||
return d['domain'][0]['id']
|
|
||||||
|
|
||||||
def _get_all_accounts(self, domain_id: str) -> List[Dict]:
|
|
||||||
url = self._create_url({
|
|
||||||
'command': 'listAccounts',
|
|
||||||
'domainid': domain_id,
|
|
||||||
'details': 'min',
|
|
||||||
})
|
|
||||||
resp = requests.get(url)
|
|
||||||
resp.raise_for_status()
|
|
||||||
d = resp.json()['listaccountsresponse']
|
|
||||||
if 'account' not in d:
|
|
||||||
# The API returns an empty dict if there are no accounts
|
|
||||||
return []
|
|
||||||
return d['account']
|
|
||||||
|
|
||||||
def _delete_account(self, account_id: str):
|
|
||||||
url = self._create_url({
|
|
||||||
'command': 'deleteAccount',
|
|
||||||
'id': account_id,
|
|
||||||
})
|
|
||||||
resp = requests.post(url)
|
|
||||||
resp.raise_for_status()
|
|
||||||
|
|
||||||
def create_account(self, user: IUser):
|
|
||||||
domain_id = self._get_domain_id(self.members_domain)
|
|
||||||
|
|
||||||
url = self._create_url({
|
|
||||||
'command': 'ldapCreateAccount',
|
|
||||||
'accounttype': '0',
|
|
||||||
'domainid': domain_id,
|
|
||||||
'username': user.uid,
|
|
||||||
})
|
|
||||||
resp = requests.post(url)
|
|
||||||
d = resp.json()['createaccountresponse']
|
|
||||||
if not resp.ok:
|
|
||||||
raise CloudStackAPIError(d['errortext'])
|
|
||||||
|
|
||||||
def purge_accounts(self) -> Dict:
|
|
||||||
accounts_deleted = []
|
|
||||||
accounts_to_be_deleted = []
|
|
||||||
result = {
|
|
||||||
'accounts_deleted': accounts_deleted,
|
|
||||||
'accounts_to_be_deleted': accounts_to_be_deleted,
|
|
||||||
}
|
|
||||||
|
|
||||||
current_term = Term.current()
|
|
||||||
beginning_of_term = current_term.to_datetime()
|
|
||||||
now = utils.get_current_datetime()
|
|
||||||
delta = now - beginning_of_term
|
|
||||||
if delta.days < 30:
|
|
||||||
# one-month grace period
|
|
||||||
return result
|
|
||||||
|
|
||||||
ldap_srv = component.getUtility(ILDAPService)
|
|
||||||
mail_srv = component.getUtility(IMailService)
|
|
||||||
k8s_srv = component.getUtility(IKubernetesService)
|
|
||||||
domain_id = self._get_domain_id(self.members_domain)
|
|
||||||
accounts = self._get_all_accounts(domain_id)
|
|
||||||
|
|
||||||
if os.path.isfile(self.pending_deletions_file):
|
|
||||||
state = json.load(open(self.pending_deletions_file))
|
|
||||||
last_check = datetime.datetime.fromtimestamp(state['timestamp'])
|
|
||||||
delta = now - last_check
|
|
||||||
if delta.days < 7:
|
|
||||||
logger.debug(
|
|
||||||
'Skipping account purge because less than one week has '
|
|
||||||
'passed since the warning emails were sent out'
|
|
||||||
)
|
|
||||||
accounts_to_be_deleted.extend(state['accounts_to_be_deleted'])
|
|
||||||
return result
|
|
||||||
username_to_account_id = {
|
|
||||||
account['name']: account['id']
|
|
||||||
for account in accounts
|
|
||||||
}
|
|
||||||
for username in state['accounts_to_be_deleted']:
|
|
||||||
if username not in username_to_account_id:
|
|
||||||
continue
|
|
||||||
user = ldap_srv.get_user(username)
|
|
||||||
if user.membership_is_valid():
|
|
||||||
continue
|
|
||||||
account_id = username_to_account_id[username]
|
|
||||||
|
|
||||||
# Delete CloudStack resources
|
|
||||||
self._delete_account(account_id)
|
|
||||||
# Delete NGINX resources
|
|
||||||
self.vhost_mgr.delete_all_vhosts_for_user(username)
|
|
||||||
# Delete k8s resources
|
|
||||||
k8s_srv.delete_account(username)
|
|
||||||
|
|
||||||
accounts_deleted.append(username)
|
|
||||||
mail_srv.send_cloud_account_has_been_deleted_message(user)
|
|
||||||
logger.info(f'Deleted cloud account for {username}')
|
|
||||||
os.unlink(self.pending_deletions_file)
|
|
||||||
return result
|
|
||||||
|
|
||||||
state = {
|
|
||||||
'timestamp': int(now.timestamp()),
|
|
||||||
'accounts_to_be_deleted': accounts_to_be_deleted,
|
|
||||||
}
|
|
||||||
for account in accounts:
|
|
||||||
username = account['name']
|
|
||||||
account_id = account['id']
|
|
||||||
user = ldap_srv.get_user(username)
|
|
||||||
if user.membership_is_valid():
|
|
||||||
continue
|
|
||||||
accounts_to_be_deleted.append(username)
|
|
||||||
mail_srv.send_cloud_account_will_be_deleted_message(user)
|
|
||||||
logger.info(
|
|
||||||
f'A warning email was sent to {username} because their '
|
|
||||||
'cloud account will be deleted'
|
|
||||||
)
|
|
||||||
if accounts_to_be_deleted:
|
|
||||||
json.dump(state, open(self.pending_deletions_file, 'w'))
|
|
||||||
return result
|
|
||||||
|
|
||||||
def _check_rate_limit(self, username: str):
|
|
||||||
if os.path.exists(self.vhost_rate_limit_file):
|
|
||||||
d = json.load(open(self.vhost_rate_limit_file))
|
|
||||||
else:
|
|
||||||
d = {}
|
|
||||||
now = int(utils.get_current_datetime().timestamp())
|
|
||||||
if username not in d:
|
|
||||||
return
|
|
||||||
time_passed = now - d[username]
|
|
||||||
if time_passed < self.vhost_rate_limit_secs:
|
|
||||||
time_remaining = self.vhost_rate_limit_secs - time_passed
|
|
||||||
raise RateLimitError(f'Please wait {time_remaining} seconds')
|
|
||||||
|
|
||||||
def _update_rate_limit_timestamp(self, username: str):
|
|
||||||
if os.path.exists(self.vhost_rate_limit_file):
|
|
||||||
d = json.load(open(self.vhost_rate_limit_file))
|
|
||||||
else:
|
|
||||||
d = {}
|
|
||||||
now = int(utils.get_current_datetime().timestamp())
|
|
||||||
d[username] = now
|
|
||||||
json.dump(d, open(self.vhost_rate_limit_file, 'w'))
|
|
||||||
|
|
||||||
def create_vhost(self, username: str, domain: str, ip_address: str):
|
|
||||||
if self.vhost_mgr.get_num_vhosts(username) >= self.max_vhosts_per_account:
|
|
||||||
raise Exception(f'Only {self.max_vhosts_per_account} vhosts '
|
|
||||||
'allowed per account')
|
|
||||||
if not self.vhost_mgr.is_valid_domain(username, domain):
|
|
||||||
raise InvalidDomainError()
|
|
||||||
if not self.vhost_mgr.is_valid_ip_address(ip_address):
|
|
||||||
raise InvalidIPError()
|
|
||||||
self._check_rate_limit(username)
|
|
||||||
# Wait for the vhost creation to succeed before updating the timestamp;
|
|
||||||
# we don't want to force people to wait if they had a typo in their
|
|
||||||
# domain or something.
|
|
||||||
self.vhost_mgr.create_vhost(username, domain, ip_address)
|
|
||||||
self._update_rate_limit_timestamp(username)
|
|
||||||
|
|
||||||
def delete_vhost(self, username: str, domain: str):
|
|
||||||
if not self.vhost_mgr.is_valid_domain(username, domain):
|
|
||||||
raise InvalidDomainError()
|
|
||||||
self.vhost_mgr.delete_vhost(username, domain)
|
|
||||||
|
|
||||||
def get_vhosts(self, username: str) -> List[Dict]:
|
|
||||||
return self.vhost_mgr.get_vhosts(username)
|
|
|
@ -0,0 +1,100 @@
|
||||||
|
from base64 import b64encode
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
from typing import Dict
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from zope import component
|
||||||
|
from zope.interface import implementer
|
||||||
|
|
||||||
|
from ceo_common.errors import CloudStackAPIError
|
||||||
|
from ceo_common.logger_factory import logger_factory
|
||||||
|
from ceo_common.interfaces import ICloudStackService, IConfig, IUser
|
||||||
|
|
||||||
|
logger = logger_factory(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@implementer(ICloudStackService)
|
||||||
|
class CloudStackService:
|
||||||
|
def __init__(self):
|
||||||
|
cfg = component.getUtility(IConfig)
|
||||||
|
self.api_key = cfg.get('cloudstack_api_key')
|
||||||
|
self.secret_key = cfg.get('cloudstack_secret_key')
|
||||||
|
self.base_url = cfg.get('cloudstack_base_url')
|
||||||
|
self.members_domain = 'Members'
|
||||||
|
self._cached_domain_id = None
|
||||||
|
|
||||||
|
def _create_url(self, params: Dict[str, str]) -> str:
|
||||||
|
# See https://docs.cloudstack.apache.org/en/latest/developersguide/dev.html#the-cloudstack-api
|
||||||
|
if 'apiKey' not in params and 'apikey' not in params:
|
||||||
|
params['apiKey'] = self.api_key
|
||||||
|
params['response'] = 'json'
|
||||||
|
request_str = '&'.join(
|
||||||
|
key + '=' + quote(val)
|
||||||
|
for key, val in params.items()
|
||||||
|
)
|
||||||
|
sig_str = '&'.join(
|
||||||
|
key.lower() + '=' + quote(val).lower()
|
||||||
|
for key, val in sorted(params.items())
|
||||||
|
)
|
||||||
|
sig = hmac.new(self.secret_key.encode(), sig_str.encode(), hashlib.sha1).digest()
|
||||||
|
encoded_sig = b64encode(sig).decode()
|
||||||
|
url = self.base_url + '?' + request_str + '&signature=' + quote(encoded_sig)
|
||||||
|
return url
|
||||||
|
|
||||||
|
def _get_domain_id(self) -> str:
|
||||||
|
if self._cached_domain_id is not None:
|
||||||
|
return self._cached_domain_id
|
||||||
|
url = self._create_url({
|
||||||
|
'command': 'listDomains',
|
||||||
|
'details': 'min',
|
||||||
|
'name': self.members_domain,
|
||||||
|
})
|
||||||
|
resp = requests.get(url)
|
||||||
|
resp.raise_for_status()
|
||||||
|
d = resp.json()['listdomainsresponse']
|
||||||
|
assert d['count'] == 1, 'there should be one domain found'
|
||||||
|
domain_id = d['domain'][0]['id']
|
||||||
|
self._cached_domain_id = domain_id
|
||||||
|
return domain_id
|
||||||
|
|
||||||
|
def get_accounts(self) -> Dict[str, str]:
|
||||||
|
domain_id = self._get_domain_id()
|
||||||
|
url = self._create_url({
|
||||||
|
'command': 'listAccounts',
|
||||||
|
'domainid': domain_id,
|
||||||
|
'details': 'min',
|
||||||
|
})
|
||||||
|
resp = requests.get(url)
|
||||||
|
resp.raise_for_status()
|
||||||
|
d = resp.json()['listaccountsresponse']
|
||||||
|
if 'account' not in d:
|
||||||
|
# The API returns an empty dict if there are no accounts
|
||||||
|
return []
|
||||||
|
return {
|
||||||
|
account['name']: account['id']
|
||||||
|
for account in d['account']
|
||||||
|
}
|
||||||
|
|
||||||
|
def delete_account(self, account_id: str):
|
||||||
|
url = self._create_url({
|
||||||
|
'command': 'deleteAccount',
|
||||||
|
'id': account_id,
|
||||||
|
})
|
||||||
|
resp = requests.post(url)
|
||||||
|
resp.raise_for_status()
|
||||||
|
|
||||||
|
def create_account(self, user: IUser):
|
||||||
|
domain_id = self._get_domain_id()
|
||||||
|
|
||||||
|
url = self._create_url({
|
||||||
|
'command': 'ldapCreateAccount',
|
||||||
|
'accounttype': '0',
|
||||||
|
'domainid': domain_id,
|
||||||
|
'username': user.uid,
|
||||||
|
})
|
||||||
|
resp = requests.post(url)
|
||||||
|
d = resp.json()['createaccountresponse']
|
||||||
|
if not resp.ok:
|
||||||
|
raise CloudStackAPIError(d['errortext'])
|
|
@ -1,4 +1,5 @@
|
||||||
import base64
|
import base64
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
|
@ -14,6 +15,8 @@ from ceo_common.interfaces import IConfig, IKubernetesService
|
||||||
|
|
||||||
@implementer(IKubernetesService)
|
@implementer(IKubernetesService)
|
||||||
class KubernetesService:
|
class KubernetesService:
|
||||||
|
namespace_prefix = 'csc-'
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
cfg = component.getUtility(IConfig)
|
cfg = component.getUtility(IConfig)
|
||||||
self.members_clusterrole = cfg.get('k8s_members_clusterrole')
|
self.members_clusterrole = cfg.get('k8s_members_clusterrole')
|
||||||
|
@ -30,9 +33,14 @@ class KubernetesService:
|
||||||
def _apply_manifest(self, manifest: str):
|
def _apply_manifest(self, manifest: str):
|
||||||
self._run(['kubectl', 'apply', '-f', '-'], input=manifest)
|
self._run(['kubectl', 'apply', '-f', '-'], input=manifest)
|
||||||
|
|
||||||
@staticmethod
|
@classmethod
|
||||||
def _get_namespace(username: str) -> str:
|
def _get_namespace(cls, username: str) -> str:
|
||||||
return 'csc-' + username
|
return cls.namespace_prefix + username
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_username_from_namespace(cls, namespace: str) -> str:
|
||||||
|
assert namespace.startswith(cls.namespace_prefix)
|
||||||
|
return namespace[len(cls.namespace_prefix):]
|
||||||
|
|
||||||
def create_account(self, username: str) -> str:
|
def create_account(self, username: str) -> str:
|
||||||
with tempfile.TemporaryDirectory() as tempdir:
|
with tempfile.TemporaryDirectory() as tempdir:
|
||||||
|
@ -53,23 +61,19 @@ class KubernetesService:
|
||||||
# Approve the CSR
|
# Approve the CSR
|
||||||
self._run(['kubectl', 'certificate', 'approve', csr_name])
|
self._run(['kubectl', 'certificate', 'approve', csr_name])
|
||||||
# Wait until the certificate is issued
|
# Wait until the certificate is issued
|
||||||
|
encoded_cert = ''
|
||||||
max_tries = 5
|
max_tries = 5
|
||||||
for i in range(max_tries):
|
for i in range(max_tries):
|
||||||
proc = self._run([
|
proc = self._run([
|
||||||
'kubectl', 'get', 'csr', csr_name,
|
'kubectl', 'get', 'csr', csr_name,
|
||||||
'-o', 'jsonpath={.status.conditions[0].type}',
|
'-o', 'jsonpath={.status.certificate}',
|
||||||
], capture_output=True)
|
], capture_output=True)
|
||||||
if proc.stdout == 'Approved':
|
encoded_cert = proc.stdout
|
||||||
|
if encoded_cert != '':
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
if i == max_tries:
|
if encoded_cert == '':
|
||||||
raise Exception('Timed out waiting for certificate to get issued')
|
raise Exception('Timed out waiting for certificate to get issued')
|
||||||
# Retrieve the certificate
|
|
||||||
proc = self._run([
|
|
||||||
'kubectl', 'get', 'csr', csr_name,
|
|
||||||
'-o', 'jsonpath={.status.certificate}',
|
|
||||||
], capture_output=True)
|
|
||||||
encoded_cert = proc.stdout
|
|
||||||
# Delete the CSR
|
# Delete the CSR
|
||||||
self._run(['kubectl', 'delete', 'csr', csr_name])
|
self._run(['kubectl', 'delete', 'csr', csr_name])
|
||||||
|
|
||||||
|
@ -98,3 +102,14 @@ class KubernetesService:
|
||||||
namespace = self._get_namespace(username)
|
namespace = self._get_namespace(username)
|
||||||
# don't check exit code because namespace might not exist
|
# don't check exit code because namespace might not exist
|
||||||
self._run(['kubectl', 'delete', 'namespace', namespace], check=False)
|
self._run(['kubectl', 'delete', 'namespace', namespace], check=False)
|
||||||
|
|
||||||
|
def get_accounts(self) -> List[str]:
|
||||||
|
proc = self._run(['kubectl', 'get', 'namespaces', '-o', 'json'],
|
||||||
|
capture_output=True)
|
||||||
|
items = json.loads(proc.stdout)['items']
|
||||||
|
namespaces = [item['metadata']['name'] for item in items]
|
||||||
|
return [
|
||||||
|
self._get_username_from_namespace(namespace)
|
||||||
|
for namespace in namespaces
|
||||||
|
if namespace.startswith(self.namespace_prefix)
|
||||||
|
]
|
||||||
|
|
|
@ -10,6 +10,8 @@ import jinja2
|
||||||
from zope import component
|
from zope import component
|
||||||
from zope.interface import implementer
|
from zope.interface import implementer
|
||||||
|
|
||||||
|
from .utils import rate_limit
|
||||||
|
from ceo_common.errors import InvalidDomainError, InvalidIPError
|
||||||
from ceo_common.logger_factory import logger_factory
|
from ceo_common.logger_factory import logger_factory
|
||||||
from ceo_common.interfaces import IVHostManager, IConfig
|
from ceo_common.interfaces import IVHostManager, IConfig
|
||||||
|
|
||||||
|
@ -48,6 +50,7 @@ class VHostManager:
|
||||||
self.k8s_ssl_cert = cfg.get('cloud vhosts_k8s_ssl_cert')
|
self.k8s_ssl_cert = cfg.get('cloud vhosts_k8s_ssl_cert')
|
||||||
self.k8s_ssl_key = cfg.get('cloud vhosts_k8s_ssl_key')
|
self.k8s_ssl_key = cfg.get('cloud vhosts_k8s_ssl_key')
|
||||||
|
|
||||||
|
self.max_vhosts_per_account = cfg.get('cloud vhosts_max_vhosts_per_account')
|
||||||
self.vhost_ip_min = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_min'))
|
self.vhost_ip_min = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_min'))
|
||||||
self.vhost_ip_max = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_max'))
|
self.vhost_ip_max = ipaddress.ip_address(cfg.get('cloud vhosts_ip_range_max'))
|
||||||
|
|
||||||
|
@ -59,6 +62,10 @@ class VHostManager:
|
||||||
loader=jinja2.PackageLoader('ceod.model'),
|
loader=jinja2.PackageLoader('ceod.model'),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
rate_limit_secs = cfg.get('cloud vhosts_rate_limit_seconds')
|
||||||
|
self.create_vhost = \
|
||||||
|
rate_limit('create_vhost', rate_limit_secs)(self.create_vhost)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _vhost_filename(username: str, domain: str) -> str:
|
def _vhost_filename(username: str, domain: str) -> str:
|
||||||
"""Generate a filename for the vhost record"""
|
"""Generate a filename for the vhost record"""
|
||||||
|
@ -82,20 +89,25 @@ class VHostManager:
|
||||||
self._run(['systemctl', 'reload', 'nginx'])
|
self._run(['systemctl', 'reload', 'nginx'])
|
||||||
|
|
||||||
def is_valid_domain(self, username: str, domain: str) -> bool:
|
def is_valid_domain(self, username: str, domain: str) -> bool:
|
||||||
subdomain = username + '.' + self.vhost_domain
|
|
||||||
if domain == subdomain:
|
|
||||||
return True
|
|
||||||
# Members may only create domains of the form *.username.csclub.cloud
|
|
||||||
# or *-username.csclub.cloud
|
|
||||||
if not domain.endswith('.' + subdomain) and \
|
|
||||||
not domain.endswith('-' + subdomain):
|
|
||||||
return False
|
|
||||||
# Make sure that the domain doesn't have invalid characters
|
|
||||||
if VALID_DOMAIN_RE.match(domain) is None:
|
if VALID_DOMAIN_RE.match(domain) is None:
|
||||||
return False
|
return False
|
||||||
if len(domain) > 80:
|
if len(domain) > 80:
|
||||||
return False
|
return False
|
||||||
return True
|
|
||||||
|
if domain.endswith('.' + self.k8s_vhost_domain):
|
||||||
|
prefix = domain[:len(domain) - len(self.k8s_vhost_domain) - 1]
|
||||||
|
elif domain.endswith('.' + self.vhost_domain):
|
||||||
|
prefix = domain[:len(domain) - len(self.vhost_domain) - 1]
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
last_part = prefix.split('.')[-1]
|
||||||
|
|
||||||
|
if last_part == username:
|
||||||
|
return True
|
||||||
|
if last_part.endswith('-' + username):
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
def is_valid_ip_address(self, ip_address: str) -> bool:
|
def is_valid_ip_address(self, ip_address: str) -> bool:
|
||||||
if ip_address == 'k8s':
|
if ip_address == 'k8s':
|
||||||
|
@ -149,6 +161,14 @@ class VHostManager:
|
||||||
os.unlink(key_path)
|
os.unlink(key_path)
|
||||||
|
|
||||||
def create_vhost(self, username: str, domain: str, ip_address: str):
|
def create_vhost(self, username: str, domain: str, ip_address: str):
|
||||||
|
if self.get_num_vhosts(username) >= self.max_vhosts_per_account:
|
||||||
|
raise Exception(f'Only {self.max_vhosts_per_account} vhosts '
|
||||||
|
'allowed per account')
|
||||||
|
if not self.is_valid_domain(username, domain):
|
||||||
|
raise InvalidDomainError()
|
||||||
|
if not self.is_valid_ip_address(ip_address):
|
||||||
|
raise InvalidIPError()
|
||||||
|
|
||||||
cert_path, key_path = self._get_cert_and_key_path(domain)
|
cert_path, key_path = self._get_cert_and_key_path(domain)
|
||||||
if not (os.path.exists(cert_path) and os.path.exists(key_path)):
|
if not (os.path.exists(cert_path) and os.path.exists(key_path)):
|
||||||
self._acquire_new_cert(domain, cert_path, key_path)
|
self._acquire_new_cert(domain, cert_path, key_path)
|
||||||
|
@ -164,6 +184,9 @@ class VHostManager:
|
||||||
self._reload_web_server()
|
self._reload_web_server()
|
||||||
|
|
||||||
def delete_vhost(self, username: str, domain: str):
|
def delete_vhost(self, username: str, domain: str):
|
||||||
|
if not self.is_valid_domain(username, domain):
|
||||||
|
raise InvalidDomainError()
|
||||||
|
|
||||||
cert_path, key_path = self._get_cert_and_key_path(domain)
|
cert_path, key_path = self._get_cert_and_key_path(domain)
|
||||||
if cert_path not in [self.default_ssl_cert, self.k8s_ssl_cert] \
|
if cert_path not in [self.default_ssl_cert, self.k8s_ssl_cert] \
|
||||||
and cert_path.startswith(self.ssl_dir) \
|
and cert_path.startswith(self.ssl_dir) \
|
||||||
|
@ -204,3 +227,12 @@ class VHostManager:
|
||||||
logger.info(f'Deleting {filepath}')
|
logger.info(f'Deleting {filepath}')
|
||||||
os.unlink(filepath)
|
os.unlink(filepath)
|
||||||
self._reload_web_server()
|
self._reload_web_server()
|
||||||
|
|
||||||
|
def get_accounts(self) -> List[str]:
|
||||||
|
vhost_files = os.listdir(self.vhost_dir)
|
||||||
|
usernames = list({
|
||||||
|
filename.split('_', 1)[0]
|
||||||
|
for filename in vhost_files
|
||||||
|
if '_' in filename
|
||||||
|
})
|
||||||
|
return usernames
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
from .CloudService import CloudService
|
from .CloudResourceManager import CloudResourceManager
|
||||||
|
from .CloudStackService import CloudStackService
|
||||||
from .KerberosService import KerberosService
|
from .KerberosService import KerberosService
|
||||||
from .LDAPService import LDAPService, UserNotFoundError, GroupNotFoundError
|
from .LDAPService import LDAPService, UserNotFoundError, GroupNotFoundError
|
||||||
from .User import User
|
from .User import User
|
||||||
|
|
|
@ -1,6 +1,11 @@
|
||||||
from typing import Dict, List, Union
|
import functools
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from typing import Dict, List, Union, Callable
|
||||||
|
|
||||||
|
from ceo_common.errors import RateLimitError
|
||||||
from ceo_common.model import Term
|
from ceo_common.model import Term
|
||||||
|
import ceo_common.utils
|
||||||
|
|
||||||
|
|
||||||
def bytes_to_strings(data: Dict[str, List[bytes]]) -> Dict[str, List[str]]:
|
def bytes_to_strings(data: Dict[str, List[bytes]]) -> Dict[str, List[str]]:
|
||||||
|
@ -40,3 +45,56 @@ def should_be_club_rep(terms: Union[None, List[str]],
|
||||||
return True
|
return True
|
||||||
# decide using the most recent term (member or non-member)
|
# decide using the most recent term (member or non-member)
|
||||||
return max(map(Term, non_member_terms)) > max(map(Term, terms))
|
return max(map(Term, non_member_terms)) > max(map(Term, terms))
|
||||||
|
|
||||||
|
|
||||||
|
def rate_limit(api_name: str, limit_secs: int):
|
||||||
|
"""
|
||||||
|
Returns a function which returns a decorator to rate limit
|
||||||
|
an API call. Since the rate limit is per-user, the first
|
||||||
|
argument to the function being rate limited must be a username.
|
||||||
|
"""
|
||||||
|
state_dir = '/run/ceod'
|
||||||
|
if not os.path.isdir(state_dir):
|
||||||
|
os.mkdir(state_dir)
|
||||||
|
rate_limit_file = os.path.join(state_dir, 'rate_limit.json')
|
||||||
|
|
||||||
|
def _check_rate_limit(username: str):
|
||||||
|
if not os.path.exists(rate_limit_file):
|
||||||
|
return
|
||||||
|
rate_limits_by_api = json.load(open(rate_limit_file))
|
||||||
|
if api_name not in rate_limits_by_api:
|
||||||
|
return
|
||||||
|
d = rate_limits_by_api[api_name]
|
||||||
|
if username not in d:
|
||||||
|
return
|
||||||
|
now = int(ceo_common.utils.get_current_datetime().timestamp())
|
||||||
|
time_passed = now - d[username]
|
||||||
|
if time_passed < limit_secs:
|
||||||
|
time_remaining = limit_secs - time_passed
|
||||||
|
raise RateLimitError(f'Please wait {time_remaining} seconds')
|
||||||
|
|
||||||
|
def _update_rate_limit_timestamp(username: str):
|
||||||
|
if os.path.exists(rate_limit_file):
|
||||||
|
rate_limits_by_api = json.load(open(rate_limit_file))
|
||||||
|
else:
|
||||||
|
rate_limits_by_api = {}
|
||||||
|
if api_name in rate_limits_by_api:
|
||||||
|
d = rate_limits_by_api[api_name]
|
||||||
|
else:
|
||||||
|
d = {}
|
||||||
|
rate_limits_by_api[api_name] = d
|
||||||
|
now = int(ceo_common.utils.get_current_datetime().timestamp())
|
||||||
|
d[username] = now
|
||||||
|
json.dump(rate_limits_by_api, open(rate_limit_file, 'w'))
|
||||||
|
|
||||||
|
def decorator_gen(func: Callable):
|
||||||
|
@functools.wraps(func)
|
||||||
|
def decorator(username: str, *args, **kwargs):
|
||||||
|
# sanity check
|
||||||
|
assert isinstance(username, str)
|
||||||
|
_check_rate_limit(username)
|
||||||
|
func(username, *args, **kwargs)
|
||||||
|
_update_rate_limit_timestamp(username)
|
||||||
|
return decorator
|
||||||
|
|
||||||
|
return decorator_gen
|
||||||
|
|
|
@ -35,7 +35,7 @@ def test_create_account(client, mock_cloud_server, new_user, ldap_conn):
|
||||||
|
|
||||||
|
|
||||||
def test_purge_accounts(
|
def test_purge_accounts(
|
||||||
client, mock_cloud_server, cloud_srv, mock_mail_server, new_user,
|
client, mock_cloud_server, cloud_mgr, mock_mail_server, new_user,
|
||||||
ldap_conn,
|
ldap_conn,
|
||||||
):
|
):
|
||||||
uid = new_user.uid
|
uid = new_user.uid
|
||||||
|
@ -43,8 +43,8 @@ def test_purge_accounts(
|
||||||
mock_mail_server.messages.clear()
|
mock_mail_server.messages.clear()
|
||||||
accounts_deleted = []
|
accounts_deleted = []
|
||||||
accounts_to_be_deleted = []
|
accounts_to_be_deleted = []
|
||||||
if os.path.isfile(cloud_srv.pending_deletions_file):
|
if os.path.isfile(cloud_mgr.pending_deletions_file):
|
||||||
os.unlink(cloud_srv.pending_deletions_file)
|
os.unlink(cloud_mgr.pending_deletions_file)
|
||||||
expected = {
|
expected = {
|
||||||
'accounts_deleted': accounts_deleted,
|
'accounts_deleted': accounts_deleted,
|
||||||
'accounts_to_be_deleted': accounts_to_be_deleted,
|
'accounts_to_be_deleted': accounts_to_be_deleted,
|
||||||
|
@ -66,7 +66,7 @@ def test_purge_accounts(
|
||||||
status, data = client.post('/api/cloud/accounts/purge')
|
status, data = client.post('/api/cloud/accounts/purge')
|
||||||
assert status == 200
|
assert status == 200
|
||||||
assert data == expected
|
assert data == expected
|
||||||
assert os.path.isfile(cloud_srv.pending_deletions_file)
|
assert os.path.isfile(cloud_mgr.pending_deletions_file)
|
||||||
assert len(mock_mail_server.messages) == 1
|
assert len(mock_mail_server.messages) == 1
|
||||||
|
|
||||||
# user still has one week left to renew their membership
|
# user still has one week left to renew their membership
|
||||||
|
@ -164,8 +164,7 @@ def test_cloud_vhosts(cfg, client, new_user, ldap_conn):
|
||||||
|
|
||||||
|
|
||||||
def test_cloud_vhosts_purged_account(
|
def test_cloud_vhosts_purged_account(
|
||||||
cfg, client, mock_cloud_server, mock_mail_server, cloud_srv, new_user,
|
cfg, client, mock_cloud_server, mock_mail_server, new_user, ldap_conn,
|
||||||
ldap_conn,
|
|
||||||
):
|
):
|
||||||
uid = new_user.uid
|
uid = new_user.uid
|
||||||
members_domain = cfg.get('cloud vhosts_members_domain')
|
members_domain = cfg.get('cloud vhosts_members_domain')
|
||||||
|
@ -196,3 +195,11 @@ def test_cloud_vhosts_purged_account(
|
||||||
assert data == {'vhosts': []}
|
assert data == {'vhosts': []}
|
||||||
|
|
||||||
mock_mail_server.messages.clear()
|
mock_mail_server.messages.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def test_k8s_create_account(client, new_user, ldap_conn):
|
||||||
|
uid = new_user.uid
|
||||||
|
status, data = client.post('/api/cloud/k8s/accounts/create', principal=uid)
|
||||||
|
assert status == 200
|
||||||
|
assert data['status'] == 'OK'
|
||||||
|
assert 'kubeconfig' in data
|
||||||
|
|
|
@ -0,0 +1,27 @@
|
||||||
|
import json
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
|
def test_k8s_create_account(k8s_srv):
|
||||||
|
# The KubernetesService should really be tested against a live cluster,
|
||||||
|
# but that's hard to do in a local environment.
|
||||||
|
# So we're just going to make sure that it doesn't crash, at least.
|
||||||
|
k8s_srv.create_account('test1')
|
||||||
|
|
||||||
|
|
||||||
|
def test_k8s_get_accounts(k8s_srv):
|
||||||
|
class MockProcess:
|
||||||
|
def __init__(self, output: str):
|
||||||
|
self.stdout = output
|
||||||
|
|
||||||
|
def mock_run(args, check=True, **kwargs):
|
||||||
|
return MockProcess(json.dumps({'items': [
|
||||||
|
{'metadata': {'name': 'csc-test1'}},
|
||||||
|
{'metadata': {'name': 'csc-test2'}},
|
||||||
|
{'metadata': {'name': 'kube-system'}},
|
||||||
|
]}))
|
||||||
|
|
||||||
|
expected = ['test1', 'test2']
|
||||||
|
with patch.object(k8s_srv, '_run') as run_method:
|
||||||
|
run_method.side_effect = mock_run
|
||||||
|
assert k8s_srv.get_accounts() == expected
|
|
@ -1,8 +1,12 @@
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
def test_vhost_mgr(cloud_srv):
|
def test_vhost_mgr(vhost_mgr):
|
||||||
vhost_mgr = cloud_srv.vhost_mgr
|
rate_limit_file = '/run/ceod/rate_limit.json'
|
||||||
|
if os.path.exists(rate_limit_file):
|
||||||
|
os.unlink(rate_limit_file)
|
||||||
username = 'test1'
|
username = 'test1'
|
||||||
domain = username + '.csclub.cloud'
|
domain = username + '.csclub.cloud'
|
||||||
filename = f'{username}_{domain}'
|
filename = f'{username}_{domain}'
|
||||||
|
@ -17,6 +21,10 @@ def test_vhost_mgr(cloud_srv):
|
||||||
'domain': domain, 'ip_address': ip_address,
|
'domain': domain, 'ip_address': ip_address,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
|
d = json.load(open(rate_limit_file))
|
||||||
|
assert username in d['create_vhost']
|
||||||
|
os.unlink(rate_limit_file)
|
||||||
|
|
||||||
domain2 = 'app.' + domain
|
domain2 = 'app.' + domain
|
||||||
vhost_mgr.create_vhost(username, domain2, ip_address)
|
vhost_mgr.create_vhost(username, domain2, ip_address)
|
||||||
assert vhost_mgr.get_num_vhosts(username) == 2
|
assert vhost_mgr.get_num_vhosts(username) == 2
|
||||||
|
@ -26,3 +34,41 @@ def test_vhost_mgr(cloud_srv):
|
||||||
|
|
||||||
vhost_mgr.delete_all_vhosts_for_user(username)
|
vhost_mgr.delete_all_vhosts_for_user(username)
|
||||||
assert vhost_mgr.get_num_vhosts(username) == 0
|
assert vhost_mgr.get_num_vhosts(username) == 0
|
||||||
|
|
||||||
|
os.unlink(rate_limit_file)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('suffix', ['csclub.cloud', 'k8s.csclub.cloud'])
|
||||||
|
@pytest.mark.parametrize('prefix,is_valid', [
|
||||||
|
('ctdalek', True),
|
||||||
|
('ctdalek1', False),
|
||||||
|
('1ctdalek', False),
|
||||||
|
('app_ctdalek', False),
|
||||||
|
('app.ctdalek', True),
|
||||||
|
('ctdalek.app', False),
|
||||||
|
('app-ctdalek', True),
|
||||||
|
('ctdalek-app', False),
|
||||||
|
('abc.def.ctdalek', True),
|
||||||
|
('abc.def-ctdalek', True),
|
||||||
|
])
|
||||||
|
def test_vhost_domain_validation(suffix, prefix, is_valid, vhost_mgr):
|
||||||
|
username = 'ctdalek'
|
||||||
|
domain = prefix + '.' + suffix
|
||||||
|
assert vhost_mgr.is_valid_domain(username, domain) == is_valid
|
||||||
|
|
||||||
|
|
||||||
|
def test_vhost_domain_validation_2(vhost_mgr):
|
||||||
|
assert not vhost_mgr.is_valid_domain('ctdalek', 'ctdalek.csclub.internal')
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('ip_address,is_valid', [
|
||||||
|
('8.8.8.8', False),
|
||||||
|
('172.19.134.11', True),
|
||||||
|
('172.19.134.11:8000', True),
|
||||||
|
('172.19.134.1', False),
|
||||||
|
('172.19.134.254', False),
|
||||||
|
('172.19.134.254:8000', False),
|
||||||
|
('k8s', True),
|
||||||
|
])
|
||||||
|
def test_vhost_ip_validation(ip_address, is_valid, vhost_mgr):
|
||||||
|
assert vhost_mgr.is_valid_ip_address(ip_address) == is_valid
|
||||||
|
|
|
@ -25,13 +25,14 @@ from .utils import ( # noqa: F401
|
||||||
)
|
)
|
||||||
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
|
from ceo_common.interfaces import IConfig, IKerberosService, ILDAPService, \
|
||||||
IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
|
IFileService, IMailmanService, IHTTPClient, IUWLDAPService, IMailService, \
|
||||||
IDatabaseService, ICloudService, IKubernetesService
|
IDatabaseService, ICloudStackService, IKubernetesService, IVHostManager, \
|
||||||
|
ICloudResourceManager
|
||||||
from ceo_common.model import Config, HTTPClient, Term
|
from ceo_common.model import Config, HTTPClient, Term
|
||||||
from ceod.api import create_app
|
from ceod.api import create_app
|
||||||
from ceod.db import MySQLService, PostgreSQLService
|
from ceod.db import MySQLService, PostgreSQLService
|
||||||
from ceod.model import KerberosService, LDAPService, FileService, User, \
|
from ceod.model import KerberosService, LDAPService, FileService, User, \
|
||||||
MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \
|
MailmanService, Group, UWLDAPService, UWLDAPRecord, MailService, \
|
||||||
CloudService, KubernetesService
|
CloudStackService, KubernetesService, VHostManager, CloudResourceManager
|
||||||
from .MockSMTPServer import MockSMTPServer
|
from .MockSMTPServer import MockSMTPServer
|
||||||
from .MockMailmanServer import MockMailmanServer
|
from .MockMailmanServer import MockMailmanServer
|
||||||
from .MockCloudStackServer import MockCloudStackServer
|
from .MockCloudStackServer import MockCloudStackServer
|
||||||
|
@ -280,17 +281,31 @@ def vhost_dir_setup(cfg):
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope='session')
|
@pytest.fixture(scope='session')
|
||||||
def cloud_srv(cfg, vhost_dir_setup):
|
def vhost_mgr(cfg, vhost_dir_setup):
|
||||||
_cloud_srv = CloudService()
|
mgr = VHostManager()
|
||||||
component.getGlobalSiteManager().registerUtility(_cloud_srv, ICloudService)
|
component.getGlobalSiteManager().registerUtility(mgr, IVHostManager)
|
||||||
return _cloud_srv
|
return mgr
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='session')
|
||||||
|
def cloudstack_srv(cfg):
|
||||||
|
srv = CloudStackService()
|
||||||
|
component.getGlobalSiteManager().registerUtility(srv, ICloudStackService)
|
||||||
|
return srv
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope='session')
|
@pytest.fixture(scope='session')
|
||||||
def k8s_srv(cfg, vhost_dir_setup):
|
def k8s_srv(cfg, vhost_dir_setup):
|
||||||
_k8s_srv = KubernetesService()
|
srv = KubernetesService()
|
||||||
component.getGlobalSiteManager().registerUtility(_k8s_srv, IKubernetesService)
|
component.getGlobalSiteManager().registerUtility(srv, IKubernetesService)
|
||||||
return _k8s_srv
|
return srv
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='session')
|
||||||
|
def cloud_mgr(cfg):
|
||||||
|
mgr = CloudResourceManager()
|
||||||
|
component.getGlobalSiteManager().registerUtility(mgr, ICloudResourceManager)
|
||||||
|
return mgr
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True, scope='session')
|
@pytest.fixture(autouse=True, scope='session')
|
||||||
|
@ -304,8 +319,10 @@ def app(
|
||||||
mail_srv,
|
mail_srv,
|
||||||
mysql_srv,
|
mysql_srv,
|
||||||
postgresql_srv,
|
postgresql_srv,
|
||||||
cloud_srv,
|
cloudstack_srv,
|
||||||
|
vhost_mgr,
|
||||||
k8s_srv,
|
k8s_srv,
|
||||||
|
cloud_mgr,
|
||||||
):
|
):
|
||||||
app = create_app({'TESTING': True})
|
app = create_app({'TESTING': True})
|
||||||
return app
|
return app
|
||||||
|
|
Loading…
Reference in New Issue