From 96cb2bc80861461d06b584895c7979668ee3d8e8 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Tue, 3 Aug 2021 14:09:07 +0000 Subject: [PATCH] add updateprograms --- ceo_common/interfaces/ILDAPService.py | 24 +++++++ ceo_common/interfaces/IUWLDAPService.py | 14 +++- ceod/api/utils.py | 5 ++ ceod/api/uwldap.py | 23 ++++++- ceod/model/LDAPService.py | 64 ++++++++++++++++++- ceod/model/UWLDAPRecord.py | 27 ++++++-- ceod/model/UWLDAPService.py | 22 ++++++- .../uwldap/UpdateProgramsTransaction.py | 20 ++++++ ceod/transactions/uwldap/__init__.py | 1 + 9 files changed, 185 insertions(+), 15 deletions(-) create mode 100644 ceod/transactions/uwldap/UpdateProgramsTransaction.py create mode 100644 ceod/transactions/uwldap/__init__.py diff --git a/ceo_common/interfaces/ILDAPService.py b/ceo_common/interfaces/ILDAPService.py index 75d1ec77c..5641c95af 100644 --- a/ceo_common/interfaces/ILDAPService.py +++ b/ceo_common/interfaces/ILDAPService.py @@ -1,3 +1,5 @@ +from typing import List, Union + from zope.interface import Interface from .IUser import IUser @@ -42,3 +44,25 @@ class ILDAPService(Interface): def remove_sudo_role(uid: str): """Remove the sudo role for this club from the database.""" + + def update_programs( + dry_run: bool = False, + members: Union[List[str], None] = None, + ): + """ + Sync the 'program' attribute in CSC LDAP with UW LDAP. + If `dry_run` is set to True, then a list of members whose programs + *would* be changed is returned along with their old and new programs: + ``` + [ + ('user1', 'old_program1', 'new_program1'), + ('user2', 'old_program2', 'new_program2'), + ... + ] + ``` + If `members` is set to a list of usernames, then only + those members will (possibly) have their programs updated. + On success, a list of members whose programs *were* changed will + be returned along with their new programs, in the same format + described above. + """ diff --git a/ceo_common/interfaces/IUWLDAPService.py b/ceo_common/interfaces/IUWLDAPService.py index 72c66e808..01c913e56 100644 --- a/ceo_common/interfaces/IUWLDAPService.py +++ b/ceo_common/interfaces/IUWLDAPService.py @@ -1,4 +1,4 @@ -from typing import Union +from typing import List, Union from zope.interface import Interface @@ -6,10 +6,20 @@ from zope.interface import Interface class IUWLDAPService(Interface): """Represents the UW LDAP database.""" - def get(username: str): + def get_user(username: str): """ Return the LDAP record for the given user, or None if no such record exists. :rtype: Union[UWLDAPRecord, None] """ + + def get_programs_for_users(usernames: List[str]) -> List[Union[str, None]]: + """ + Return the programs for the given users from UWLDAP. + If no record or program is found for a user, their entry in + the returned list will be None. + Example: + get_programs_for_users(['user1', 'user2', 'user3']) + -> ['program1', None, 'program2'] + """ diff --git a/ceod/api/utils.py b/ceod/api/utils.py index d8cb15a7f..d50b055f9 100644 --- a/ceod/api/utils.py +++ b/ceod/api/utils.py @@ -7,6 +7,7 @@ import traceback from typing import Callable, List from flask import current_app +from flask.json import jsonify from flask_kerberos import requires_authentication from ceo_common.logger_factory import logger_factory @@ -111,6 +112,10 @@ def create_sync_response(txn: AbstractTransaction): """ try: txn.execute() + # if the result is already an Object, don't wrap it again + if isinstance(txn.result, dict) or isinstance(txn.result, list): + return jsonify(txn.result) + # if the result is a string or number, wrap it in an Object return {'result': txn.result} except Exception as err: logger.warning('Transaction failed:\n' + traceback.format_exc()) diff --git a/ceod/api/uwldap.py b/ceod/api/uwldap.py index 39c2e03be..93882dc0b 100644 --- a/ceod/api/uwldap.py +++ b/ceod/api/uwldap.py @@ -1,7 +1,10 @@ -from flask import Blueprint +from flask import Blueprint, request +from flask.json import jsonify from zope import component -from ceo_common.interfaces import IUWLDAPService +from .utils import create_sync_response, authz_restrict_to_syscom +from ceo_common.interfaces import IUWLDAPService, ILDAPService +from ceod.transactions.uwldap import UpdateProgramsTransaction bp = Blueprint('uwldap', __name__) @@ -9,9 +12,23 @@ bp = Blueprint('uwldap', __name__) @bp.route('/') def get_user(username: str): uwldap_srv = component.getUtility(IUWLDAPService) - record = uwldap_srv.get(username) + record = uwldap_srv.get_user(username) if record is None: return { 'error': 'user not found', }, 404 return record.to_dict() + + +@bp.route('/updateprograms', methods=['POST']) +@authz_restrict_to_syscom +def update_programs(): + ldap_srv = component.getUtility(ILDAPService) + body = request.get_json(force=True) + members = body.get('members') + if body.get('dry_run'): + return jsonify( + ldap_srv.update_programs(dry_run=True, members=members) + ) + txn = UpdateProgramsTransaction(members=members) + return create_sync_response(txn) diff --git a/ceod/model/LDAPService.py b/ceod/model/LDAPService.py index 325f8725c..c9c0fe7b4 100644 --- a/ceod/model/LDAPService.py +++ b/ceod/model/LDAPService.py @@ -1,6 +1,7 @@ import copy import grp import pwd +from typing import Union, List import ldap import ldap.modlist @@ -8,10 +9,12 @@ from zope import component from zope.interface import implementer from ceo_common.errors import UserNotFoundError, GroupNotFoundError -from ceo_common.interfaces import ILDAPService, IKerberosService, IConfig, IUser, IGroup +from ceo_common.interfaces import ILDAPService, IKerberosService, IConfig, \ + IUser, IGroup, IUWLDAPService from .User import User from .Group import Group from .SudoRole import SudoRole +from .utils import dn_to_uid, bytes_to_strings @implementer(ILDAPService) @@ -49,7 +52,7 @@ class LDAPService: def get_user(self, username: str) -> IUser: conn = self._get_ldap_conn(False) - base = f'uid={username},{self.ldap_users_base}' + base = self.uid_to_dn(username) try: _, result = conn.search_s(base, ldap.SCOPE_BASE)[0] return User.deserialize_from_ldap(result) @@ -58,13 +61,19 @@ class LDAPService: def get_group(self, cn: str) -> IGroup: conn = self._get_ldap_conn(False) - base = f'cn={cn},{self.ldap_groups_base}' + base = self.group_cn_to_dn(cn) try: _, result = conn.search_s(base, ldap.SCOPE_BASE)[0] return Group.deserialize_from_ldap(result) except ldap.NO_SUCH_OBJECT: raise GroupNotFoundError() + def uid_to_dn(self, uid: str): + return f'uid={uid},{self.ldap_users_base}' + + def group_cn_to_dn(self, cn: str): + return f'cn={cn},{self.ldap_groups_base}' + def _get_next_uid(self, conn: ldap.ldapobject.LDAPObject, min_id: int, max_id: int) -> int: """Gets the next available UID number between min_id and max_id, inclusive.""" def uid_exists(uid: int) -> bool: @@ -152,3 +161,52 @@ class LDAPService: def remove_group(self, group: IGroup): conn = self._get_ldap_conn() conn.delete_s(group.dn) + + def update_programs( + self, + dry_run: bool = False, + members: Union[List[str], None] = None, + uwldap_batch_size: int = 100, + ): + if members: + filter_str = '(|' + ''.join([ + f'(uid={uid})' for uid in members + ]) + ')' + else: + filter_str = None + conn = self._get_ldap_conn() + raw_csc_records = conn.search_s( + self.ldap_users_base, ldap.SCOPE_SUBTREE, filter_str, + attrlist=['program']) + uids = [ + dn_to_uid(dn) for dn, _ in raw_csc_records + ] + csc_programs = [ + bytes_to_strings(data).get('program', [None])[0] + for _, data in raw_csc_records + ] + + uwldap_srv = component.getUtility(IUWLDAPService) + uw_programs = [] + # send queries in small batches + for i in range(0, len(csc_programs), uwldap_batch_size): + batch_uids = uids[i:i + uwldap_batch_size] + batch_uw_programs = uwldap_srv.get_programs_for_users(batch_uids) + uw_programs.extend(batch_uw_programs) + users_to_change = [ + (uids[i], csc_programs[i], uw_programs[i]) + for i in range(len(uids)) + if csc_programs[i] != uw_programs[i] and ( + uw_programs[i] not in (None, 'expired', 'orphaned') + ) + ] + if dry_run: + return users_to_change + + for uid, old_program, new_program in users_to_change: + old_entry = {'program': [old_program.encode()]} + new_entry = {'program': [new_program.encode()]} + modlist = ldap.modlist.modifyModlist(old_entry, new_entry) + conn.modify_s(self.uid_to_dn(uid), modlist) + + return users_to_change diff --git a/ceod/model/UWLDAPRecord.py b/ceod/model/UWLDAPRecord.py index 90aceda55..2c2468caf 100644 --- a/ceod/model/UWLDAPRecord.py +++ b/ceod/model/UWLDAPRecord.py @@ -9,12 +9,18 @@ class UWLDAPRecord: def __init__( self, uid: str, - program: Union[str, None], mail_local_addresses: List[str], + program: Union[str, None] = None, + cn: Union[str, None] = None, + sn: Union[str, None] = None, + given_name: Union[str, None] = None, ): self.uid = uid - self.program = program self.mail_local_addresses = mail_local_addresses + self.program = program + self.cn = cn + self.sn = sn + self.given_name = given_name @staticmethod def deserialize_from_ldap(data: Dict[str, List[bytes]]): @@ -25,13 +31,24 @@ class UWLDAPRecord: data = bytes_to_strings(data) return UWLDAPRecord( uid=data['uid'][0], - program=data.get('ou', [None])[0], mail_local_addresses=data['mailLocalAddress'], + program=data.get('ou', [None])[0], + cn=data.get('cn', [None])[0], + sn=data.get('sn', [None])[0], + given_name=data.get('givenName', [None])[0], ) def to_dict(self): - return { + data = { 'uid': self.uid, - 'program': self.program, 'mail_local_addresses': self.mail_local_addresses, } + if self.program: + data['program'] = self.program + if self.cn: + data['cn'] = self.cn + if self.sn: + data['sn'] = self.sn + if self.given_name: + data['given_name'] = self.given_name + return data diff --git a/ceod/model/UWLDAPService.py b/ceod/model/UWLDAPService.py index f1caec4ad..77308ee9e 100644 --- a/ceod/model/UWLDAPService.py +++ b/ceod/model/UWLDAPService.py @@ -1,10 +1,11 @@ -from typing import Union +from typing import Union, List import ldap from zope import component from zope.interface import implementer from .UWLDAPRecord import UWLDAPRecord +from .utils import dn_to_uid, bytes_to_strings from ceo_common.interfaces import IUWLDAPService, IConfig @@ -15,10 +16,27 @@ class UWLDAPService: self.uwldap_server_url = cfg.get('uwldap_server_url') self.uwldap_base = cfg.get('uwldap_base') - def get(self, username: str) -> Union[UWLDAPRecord, None]: + def get_user(self, username: str) -> Union[UWLDAPRecord, None]: conn = ldap.initialize(self.uwldap_server_url) results = conn.search_s(self.uwldap_base, ldap.SCOPE_SUBTREE, f'uid={username}') if not results: return None _, data = results[0] # discard the dn return UWLDAPRecord.deserialize_from_ldap(data) + + def get_programs_for_users(self, usernames: List[str]) -> List[Union[str, None]]: + filter_str = '(|' + ''.join([f'(uid={uid})' for uid in usernames]) + ')' + programs = [None] * len(usernames) + user_indices = {uid: i for i, uid in enumerate(usernames)} + + conn = ldap.initialize(self.uwldap_server_url) + records = conn.search_s( + self.uwldap_base, ldap.SCOPE_SUBTREE, filter_str, attrlist=['ou']) + for dn, data in records: + uid = dn_to_uid(dn) + idx = user_indices[uid] + data = bytes_to_strings(data) + program = data.get('ou', [None])[0] + if program: + programs[idx] = program + return programs diff --git a/ceod/transactions/uwldap/UpdateProgramsTransaction.py b/ceod/transactions/uwldap/UpdateProgramsTransaction.py new file mode 100644 index 000000000..eae67aa42 --- /dev/null +++ b/ceod/transactions/uwldap/UpdateProgramsTransaction.py @@ -0,0 +1,20 @@ +from typing import Union, List + +from zope import component + +from ..AbstractTransaction import AbstractTransaction +from ceo_common.interfaces import ILDAPService + + +class UpdateProgramsTransaction(AbstractTransaction): + """Transaction to sync the 'program' attribute in CSC LDAP with UW LDAP.""" + + def __init__(self, members: Union[List[str], None]): + super().__init__() + self.members = members + self.ldap_srv = component.getUtility(ILDAPService) + + def child_execute_iter(self): + users_updated = self.ldap_srv.update_programs(members=self.members) + yield 'update_programs' + self.finish(users_updated) diff --git a/ceod/transactions/uwldap/__init__.py b/ceod/transactions/uwldap/__init__.py new file mode 100644 index 000000000..44f715c8b --- /dev/null +++ b/ceod/transactions/uwldap/__init__.py @@ -0,0 +1 @@ +from .UpdateProgramsTransaction import UpdateProgramsTransaction