From b4a137355925cbe26a0ab6a35402d8da365216ba Mon Sep 17 00:00:00 2001 From: Edwin Date: Sun, 23 Oct 2022 19:50:06 -0400 Subject: [PATCH 1/7] Simplify packaging --- .gitignore | 1 + Makefile | 5 +++- PACKAGING.md | 56 +++++++++++-------------------------------- debian/source/options | 2 +- package.sh | 5 ++++ 5 files changed, 25 insertions(+), 44 deletions(-) create mode 100755 package.sh diff --git a/.gitignore b/.gitignore index 84b1aba..00c9dea 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ __pycache__/ /debian/files /debian/.debhelper/ /debian/debhelper-build-stamp +/pyceo.tar.gz diff --git a/Makefile b/Makefile index 8ecb067..dbd052e 100644 --- a/Makefile +++ b/Makefile @@ -31,4 +31,7 @@ clean: rm -rf venv rm -rf debian/{ceo,ceod,ceo-common,tmp} -.PHONY: build docs clean venv install +package: + podman run -it --rm -v "$$PWD":"$$PWD" -w "$$PWD" --security-opt label=disable debian:bullseye bash package.sh + +.PHONY: build docs clean venv install package diff --git a/PACKAGING.md b/PACKAGING.md index 8c54448..31c39b1 100644 --- a/PACKAGING.md +++ b/PACKAGING.md @@ -10,53 +10,25 @@ Make sure your GPG key is in /srv/debian/gpg on potassium-benzoate. See for instructions. ## Creating the package -Use Docker/Podman to avoid screwing up your main system. -For example, to create a package for bullseye (replace `podman` with `docker` in all instances below if you're using Docker): -```sh -podman run -it --name pyceo-packaging -v "$PWD":"$PWD" -w "$PWD" debian:bullseye bash -``` -**Important**: Make sure to use a container image for the same distribution which you're packaging. +Make sure you have Podman installed. + +Update `VERSION.txt` to the next version, `debian/changelog` with the new changes and do a git commit. + +Run `make package` to build the package and create the tarball `pyceo.tar.gz` in the repo root. + +**Important**: Make sure the container image for the same distribution which you're packaging. For example, if you're creating a package for bullseye, you should be using the debian:bullseye -Docker image (this is because the virtualenv symlinks python to the OS' version of python). +Docker image (this is because the virtualenv symlinks python to the OS' version of python). Also make +sure that all of the packages in the 'Build-Depends' section in debian/control are installed in the container. -Here are some of the prerequisites you'll need to build the deb files -(run this inside the container): +At some point before actually uploading, you'll need to the sign the package. On the extracted tarball, +run the following while placing your key ID after the `-k` argument, e.g. ```sh -apt update -apt install -y devscripts debhelper git-buildpackage vim -``` -Make sure to also install all of the packages in the 'Build-Depends' section in debian/control. - -Update VERSION.txt to the next version, and do a git commit. - -Now run `dch -i` and edit the changelog. - -Now you will build a signed package. Place your key ID after the `-k` argument, e.g. -```sh -gbp buildpackage --git-upstream-branch=master -k8E5568ABB0CF96BC367806ED127923BE10DA48DC -``` -This will create a bunch of files (deb, dsc, tar.gz, etc.) in the parent directory. - -Now do another git commit (since you edited the changelog file). - -To clean the packages: -```sh -rm ../*.{xz,gz,dsc,build,buildinfo,changes,deb} +debsign -k8E5568ABB0CF96BC367806ED127923BE10DA48DC *.changes ``` ## Uploading the package -Inside the container, go up one directory, and create a tarball with all the package files: -``` -cd .. -tar zcvf pyceo.tar.gz *.{xz,gz,dsc,build,buildinfo,changes,deb} -``` -Outside of the container (i.e. on your personal machine), copy the tarball out of the -container into your current directory, e.g. -``` -podman cp pyceo-packaging:/home/max/repos/pyceo.tar.gz . -``` -(Replace `/home/max/repos` by the directory in the container with the tarball.) -Now upload the tarball to a CSC machine, e.g. +Upload the tarball to a CSC machine, e.g. ``` scp pyceo.tar.gz mannitol:~/ ``` @@ -73,7 +45,7 @@ At this point, you will need a dupload.conf file. Ask someone on syscom for a co Now upload the package to potassium-benzoate: ``` kinit -dupload *.changes +dupload --to debian.csclub *.changes ``` Now SSH into potassium-benzoate and run the following: diff --git a/debian/source/options b/debian/source/options index fdd7a05..21bc363 100644 --- a/debian/source/options +++ b/debian/source/options @@ -1 +1 @@ -extend-diff-ignore = "^(venv/|docs/.+\.\d$|dist/|.+\.egg-info/|build/|\.vscode/|\.idea/)|__pycache__/|(^|/)\..+" +extend-diff-ignore = "^(venv/|docs/.+\.\d$|dist/|.+\.egg-info/|build/|\.vscode/|\.idea/|pyceo\.tar\.gz)|__pycache__/|(^|/)\..+" diff --git a/package.sh b/package.sh new file mode 100755 index 0000000..f81beb3 --- /dev/null +++ b/package.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +apt update && apt install -y git-buildpackage debhelper libkrb5-dev libpq-dev libaugeas0 python3-dev python3-venv scdoc +gbp buildpackage --git-export-dir=/tmp/build --git-upstream-branch=master --no-sign +tar zcvf pyceo.tar.gz -C /tmp/build . From 9ff3d850c97626328b5b24a2162af4f74f67e6e1 Mon Sep 17 00:00:00 2001 From: Edwin Date: Sun, 23 Oct 2022 19:50:38 -0400 Subject: [PATCH 2/7] Release 1.0.24 --- VERSION.txt | 2 +- debian/changelog | 13 +++++++++++++ debian/control | 3 ++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index 154b9fc..79728fe 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.0.23 +1.0.24 diff --git a/debian/changelog b/debian/changelog index 457b21f..16803ed 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,16 @@ +ceo (1.0.24-bullseye1) bullseye; urgency=high + + * Add support for using number in member terms renwewal API + * Sort group member listing by WatIAM ID + * Add more logging for Cloudstack + * Use LDAP instead of NSS + * Fix shadowExpire deserialization + * Fix email formatting bug in ClubWebHostingService + * Check if mail_local_addresses exists in UWLDAP entry + * Remove override_dh_systemd_start + + -- Edwin Sun, 23 Oct 2022 00:00:00 +0000 + ceo (1.0.23-bullseye1) bullseye; urgency=high * Fix some bugs in ClubWebHostingService. diff --git a/debian/control b/debian/control index 7158ff3..d18541d 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,8 @@ Standards-Version: 4.3.0 Vcs-Git: https://git.csclub.uwaterloo.ca/public/pyceo.git Vcs-Browser: https://git.csclub.uwaterloo.ca/public/pyceo Uploaders: Max Erenberg , - Raymond Li + Raymond Li , + Edwin Build-Depends: debhelper (>= 12.1.1), python3-dev (>= 3.7), python3-venv (>= 3.7), From 2cf9e25b592aeb9f08323fd2ed918b6b6e15efa2 Mon Sep 17 00:00:00 2001 From: Edwin Date: Sun, 23 Oct 2022 21:16:58 -0400 Subject: [PATCH 3/7] More fixes --- PACKAGING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/PACKAGING.md b/PACKAGING.md index 31c39b1..7f26256 100644 --- a/PACKAGING.md +++ b/PACKAGING.md @@ -9,6 +9,8 @@ Make sure your GPG key is in /srv/debian/gpg on potassium-benzoate. See [here](https://wiki.csclub.uwaterloo.ca/Debian_Repository#Step_1:_Add_to_Uploaders) for instructions. +Make sure you are in the `csc-mirror` group too. + ## Creating the package Make sure you have Podman installed. @@ -30,7 +32,7 @@ debsign -k8E5568ABB0CF96BC367806ED127923BE10DA48DC *.changes ## Uploading the package Upload the tarball to a CSC machine, e.g. ``` -scp pyceo.tar.gz mannitol:~/ +scp pyceo.tar.gz mannitol:~ ``` SSH into that machine and extract the tarball into a separate directory: ``` From 6df1f4d4595248b2ec76c969a2d0c1ccabab84bc Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Sun, 23 Oct 2022 22:00:48 -0400 Subject: [PATCH 4/7] Revert "Simplify packaging" This reverts commit b4a137355925cbe26a0ab6a35402d8da365216ba. --- .gitignore | 1 - Makefile | 5 +--- PACKAGING.md | 60 +++++++++++++++++++++++++++++++------------ debian/source/options | 2 +- package.sh | 5 ---- 5 files changed, 46 insertions(+), 27 deletions(-) delete mode 100755 package.sh diff --git a/.gitignore b/.gitignore index 00c9dea..84b1aba 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,3 @@ __pycache__/ /debian/files /debian/.debhelper/ /debian/debhelper-build-stamp -/pyceo.tar.gz diff --git a/Makefile b/Makefile index dbd052e..8ecb067 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,4 @@ clean: rm -rf venv rm -rf debian/{ceo,ceod,ceo-common,tmp} -package: - podman run -it --rm -v "$$PWD":"$$PWD" -w "$$PWD" --security-opt label=disable debian:bullseye bash package.sh - -.PHONY: build docs clean venv install package +.PHONY: build docs clean venv install diff --git a/PACKAGING.md b/PACKAGING.md index 7f26256..62491e2 100644 --- a/PACKAGING.md +++ b/PACKAGING.md @@ -12,25 +12,53 @@ for instructions. Make sure you are in the `csc-mirror` group too. ## Creating the package -Make sure you have Podman installed. - -Update `VERSION.txt` to the next version, `debian/changelog` with the new changes and do a git commit. - -Run `make package` to build the package and create the tarball `pyceo.tar.gz` in the repo root. - -**Important**: Make sure the container image for the same distribution which you're packaging. -For example, if you're creating a package for bullseye, you should be using the debian:bullseye -Docker image (this is because the virtualenv symlinks python to the OS' version of python). Also make -sure that all of the packages in the 'Build-Depends' section in debian/control are installed in the container. - -At some point before actually uploading, you'll need to the sign the package. On the extracted tarball, -run the following while placing your key ID after the `-k` argument, e.g. +Use Docker/Podman to avoid screwing up your main system. +For example, to create a package for bullseye (replace `podman` with `docker` in all instances below if you're using Docker): ```sh -debsign -k8E5568ABB0CF96BC367806ED127923BE10DA48DC *.changes +podman run -it --name pyceo-packaging -v "$PWD":"$PWD" -w "$PWD" debian:bullseye bash +``` +**Important**: Make sure to use a container image for the same distribution which you're packaging. +For example, if you're creating a package for bullseye, you should be using the debian:bullseye +Docker image (this is because the virtualenv symlinks python to the OS' version of python). + +Here are some of the prerequisites you'll need to build the deb files +(run this inside the container): +```sh +apt update +apt install -y devscripts debhelper git-buildpackage vim +``` +Make sure to also install all of the packages in the 'Build-Depends' section in debian/control. + +Update VERSION.txt to the next version, and do a git commit. + +Now run `dch -i` and edit the changelog. + +Now you will build a signed package. Place your key ID after the `-k` argument, e.g. +```sh +gbp buildpackage --git-upstream-branch=master -k8E5568ABB0CF96BC367806ED127923BE10DA48DC +``` +This will create a bunch of files (deb, dsc, tar.gz, etc.) in the parent directory. + +Now do another git commit (since you edited the changelog file). + +To clean the packages: +```sh +rm ../*.{xz,gz,dsc,build,buildinfo,changes,deb} ``` ## Uploading the package -Upload the tarball to a CSC machine, e.g. +Inside the container, go up one directory, and create a tarball with all the package files: +``` +cd .. +tar zcvf pyceo.tar.gz *.{xz,gz,dsc,build,buildinfo,changes,deb} +``` +Outside of the container (i.e. on your personal machine), copy the tarball out of the +container into your current directory, e.g. +``` +podman cp pyceo-packaging:/home/max/repos/pyceo.tar.gz . +``` +(Replace `/home/max/repos` by the directory in the container with the tarball.) +Now upload the tarball to a CSC machine, e.g. ``` scp pyceo.tar.gz mannitol:~ ``` @@ -47,7 +75,7 @@ At this point, you will need a dupload.conf file. Ask someone on syscom for a co Now upload the package to potassium-benzoate: ``` kinit -dupload --to debian.csclub *.changes +dupload *.changes ``` Now SSH into potassium-benzoate and run the following: diff --git a/debian/source/options b/debian/source/options index 21bc363..fdd7a05 100644 --- a/debian/source/options +++ b/debian/source/options @@ -1 +1 @@ -extend-diff-ignore = "^(venv/|docs/.+\.\d$|dist/|.+\.egg-info/|build/|\.vscode/|\.idea/|pyceo\.tar\.gz)|__pycache__/|(^|/)\..+" +extend-diff-ignore = "^(venv/|docs/.+\.\d$|dist/|.+\.egg-info/|build/|\.vscode/|\.idea/)|__pycache__/|(^|/)\..+" diff --git a/package.sh b/package.sh deleted file mode 100755 index f81beb3..0000000 --- a/package.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -apt update && apt install -y git-buildpackage debhelper libkrb5-dev libpq-dev libaugeas0 python3-dev python3-venv scdoc -gbp buildpackage --git-export-dir=/tmp/build --git-upstream-branch=master --no-sign -tar zcvf pyceo.tar.gz -C /tmp/build . From 6a1fa81b82a6fefeed9b1270c87c7148ed4b91db Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Sun, 23 Oct 2022 22:04:06 -0400 Subject: [PATCH 5/7] merenber signs the packages Something went wrong when e42zhang tried to upload the packages to the mirror. reprepro kept on complaining that no distribution would accept the new package. I modified the changelog, re-signed and re-uploaded the packages, and that worked, so I'm still not sure what the problem was. --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 16803ed..dcd4680 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,7 +9,7 @@ ceo (1.0.24-bullseye1) bullseye; urgency=high * Check if mail_local_addresses exists in UWLDAP entry * Remove override_dh_systemd_start - -- Edwin Sun, 23 Oct 2022 00:00:00 +0000 + -- Max Erenberg Sun, 23 Oct 2022 21:41:00 -0400 ceo (1.0.23-bullseye1) bullseye; urgency=high From 1e452d10ce277fe0f777d3425136bcd115e313f7 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Tue, 1 Nov 2022 21:02:05 -0400 Subject: [PATCH 6/7] Assume program is Alumni if UWLDAP is missing data (#84) This PR sets 'program=Alumni' for members who either do not have an 'ou' attribute in UWLDAP, or who do not have a UWLDAP entry at all. Reviewed-on: https://git.csclub.uwaterloo.ca/public/pyceo/pulls/84 Reviewed-by: Raymond Li --- ceod/model/LDAPService.py | 15 ++++++++++----- tests/conftest.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ceod/model/LDAPService.py b/ceod/model/LDAPService.py index 7ec6388..c15f916 100644 --- a/ceod/model/LDAPService.py +++ b/ceod/model/LDAPService.py @@ -316,12 +316,12 @@ class LDAPService: self, dry_run: bool = False, members: Union[List[str], None] = None, - uwldap_batch_size: int = 10, + uwldap_batch_size: int = 100, ): if members: filter = '(|' + ''.join([f'(uid={uid})' for uid in members]) + ')' else: - filter = '(objectClass=*)' + filter = '(objectClass=member)' conn = self._get_ldap_conn() conn.search( self.ldap_users_base, filter, attributes=['uid', 'program']) @@ -336,12 +336,17 @@ class LDAPService: batch_uids = uids[i:i + uwldap_batch_size] batch_uw_programs = uwldap_srv.get_programs_for_users(batch_uids) uw_programs.extend(batch_uw_programs) + # uw_programs[i] will be None if the 'ou' attribute was not + # present in UWLDAP, or if no UWLDAP entry was found at all + for i, uw_program in enumerate(uw_programs): + if uw_program in (None, 'expired', 'orphaned'): + # If the UWLDAP record is orphaned, nonexistent, or missing + # data, assume that the member graduated + uw_programs[i] = 'Alumni' users_to_change = [ (uids[i], csc_programs[i], uw_programs[i]) for i in range(len(uids)) - if csc_programs[i] != uw_programs[i] and ( - uw_programs[i] not in (None, 'expired', 'orphaned') - ) + if csc_programs[i] != uw_programs[i] ] if dry_run: return users_to_change diff --git a/tests/conftest.py b/tests/conftest.py index e3c3c3a..208bc90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -298,6 +298,17 @@ def uwldap_srv(cfg, ldap_conn): delete_subtree(conn, base_dn) conn.add(base_dn, 'organizationalUnit') + conn.add( + f'uid=ctdalek,{base_dn}', + ['inetLocalMailRecipient', 'inetOrgPerson', 'organizationalPerson', 'person'], + { + 'mailLocalAddress': 'ctdalek@uwaterloo.internal', + 'ou': 'Math', + 'cn': 'Calum T. Dalek', + 'sn': 'Dalek', + 'givenName': 'Calum', + }, + ) _uwldap_srv = UWLDAPService() component.getGlobalSiteManager().registerUtility(_uwldap_srv, IUWLDAPService) yield _uwldap_srv From c0c97365931f5ea1d14736e97abbc01a37c4cd53 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Sun, 6 Nov 2022 15:23:27 -0500 Subject: [PATCH 7/7] Use the admin creds in the HTTPClient when necessary (#85) Currently, ceod uses the Kerberos credentials of the client when making requests to other services. This requires the client to send delegated credentials. Unfortunately the NPM krb5 package appears to be unable to perform delegation. So we will use the admin credentials instead (when appropriate). Reviewed-on: https://git.csclub.uwaterloo.ca/public/pyceo/pulls/85 Reviewed-by: Raymond Li Co-authored-by: Max Erenberg Co-committed-by: Max Erenberg --- ceo_common/model/HTTPClient.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ceo_common/model/HTTPClient.py b/ceo_common/model/HTTPClient.py index 1348a9f..cbf635b 100644 --- a/ceo_common/model/HTTPClient.py +++ b/ceo_common/model/HTTPClient.py @@ -6,7 +6,7 @@ from requests_gssapi import HTTPSPNEGOAuth from zope import component from zope.interface import implementer -from ceo_common.interfaces import IConfig, IHTTPClient +from ceo_common.interfaces import IConfig, IHTTPClient, IKerberosService @implementer(IHTTPClient) @@ -40,10 +40,18 @@ class HTTPClient: 'opportunistic_auth': True, 'target_name': gssapi.Name('ceod/' + host), } - if flask.has_request_context() and 'client_token' in g: + if flask.has_request_context(): # This is reached when we are the server and the client has # forwarded their credentials to us. - spnego_kwargs['creds'] = gssapi.Credentials(token=g.client_token) + token = None + if g.get('need_admin_creds', False): + # Some Kerberos bindings in some programming languages can't + # perform delegation, so use the admin creds here. + token = component.getUtility(IKerberosService).get_admin_creds_token() + elif 'client_token' in g: + token = g.client_token + if token is not None: + spnego_kwargs['creds'] = gssapi.Credentials(token=token) elif delegate: # This is reached when we are the client and we want to # forward our credentials to the server.