From a1da002935a1f66823bf21a22d1bea3014217537 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Sun, 21 Nov 2021 12:43:10 -0500 Subject: [PATCH] add zfssync.py and remove trailing whitespace --- roles/ftp/tasks/main.yml | 6 +- roles/mirror/tasks/main.yml | 14 ++- roles/mirror/templates/zfssync.py | 152 ++++++++++++++++++++++++++++++ roles/nginx/tasks/main.yml | 2 +- roles/ssh/tasks/main.yml | 4 +- roles/system/tasks/main.yml | 20 ++-- 6 files changed, 179 insertions(+), 19 deletions(-) create mode 100755 roles/mirror/templates/zfssync.py diff --git a/roles/ftp/tasks/main.yml b/roles/ftp/tasks/main.yml index 45f32e6..7a94a0f 100644 --- a/roles/ftp/tasks/main.yml +++ b/roles/ftp/tasks/main.yml @@ -30,9 +30,9 @@ # will not regenerated every playbook run command: cmd: > - openssl dhparam - -outform PEM -2|-5 - 1024|1536|2048|3072|4096|6144|7680|8192 + openssl dhparam + -outform PEM -2|-5 + 1024|1536|2048|3072|4096|6144|7680|8192 > /etc/proftpd/dhparams.pem creates: /etc/proftpd/dhparams.pem diff --git a/roles/mirror/tasks/main.yml b/roles/mirror/tasks/main.yml index bb3503d..a2da36f 100644 --- a/roles/mirror/tasks/main.yml +++ b/roles/mirror/tasks/main.yml @@ -6,6 +6,14 @@ group: root mode: "0644" +- name: copy zfssync python script + copy: + src: "{{ role_path }}/templates/zfssync.py" + dest: /usr/local/bin/zfssync.py + owner: root + group: root + mode: "0755" + - name: restart and enable cron systemd: name: cron @@ -21,7 +29,7 @@ mode: preserve - name: copy files for /mirror/root - copy: + copy: src: "{{ role_path }}/templates/root/root/" dest: /mirror/root owner: root @@ -54,7 +62,7 @@ - name: create symlinks from busybox and arthur # noqa deprecated-command-syntax # need chdir to create relative symlinks - command: + command: cmd: "ln -s {{ item.src }} {{ item.dest }}" chdir: "/mirror/merlin/bin" creates: "/mirror/merlin/bin/{{ item.dest }}" @@ -84,7 +92,7 @@ - name: create first merlin.sock symlink # noqa deprecated-command-syntax # need chdir to create relative symlinks - command: + command: cmd: "ln -s run/merlin.sock merlin.sock" chdir: "/mirror/merlin" creates: "/mirror/merlin/merlin.sock" diff --git a/roles/mirror/templates/zfssync.py b/roles/mirror/templates/zfssync.py new file mode 100755 index 0000000..3c2a0e7 --- /dev/null +++ b/roles/mirror/templates/zfssync.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 + +import logging, sys, os, shutil, time, datetime, yaml, socket +from weir import zfs, process + +HOLD_SYNC_TAG = 'sync' + +def setup_logging(): + logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s") + +class Project: + def __init__(self, pool, name, host=None): + self.host = host + self.pool = pool + self.name = name + + self.fs = zfs.open(self.path(), 'filesystem') + + # Check that the filesystem exists + try: + self.fs.getprops() + except process.DatasetNotFoundError: + self.fs = zfs.create(self.path(), 'filesystem') + + + def path(self): + if self.host: + return 'zfs://{}/{}/{}'.format(self.host, self.pool, self.name) + + return '{}/{}'.format(self.pool, self.name) + + def snapshots(self): + return self.fs.snapshots() + + def snapshot(self, name=None, create=True): + if name is None: + name = 'test' + + snap = list(filter(lambda s: s.snapname() == name, self.snapshots())) + if len(snap) == 0 and create: + return self.fs.snapshot(name) + elif len(snap) == 0: + return None + + return snap[0] + + + +def sync(src, dest): + # Find the last snapshot in common between src and dest + src_snapshots = map(lambda s: s.snapname(), src.snapshots()) + dest_snapshots = map(lambda s: s.snapname(), dest.snapshots()) + common_snapshots = list(set(src_snapshots) & set(dest_snapshots)) + common_snapshots.sort() + + previous_snapshot = None + if len(common_snapshots): + previous_snapshot = dict([ + ("src", src.snapshot(common_snapshots[-1], create=False)), + ("dest", dest.snapshot(common_snapshots[-1], create=False)) + ]) + + # Create the new snapshot + new_snapshot = src.snapshot(datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")) + + # Hold the source snapshot + try: + new_snapshot.hold(HOLD_SYNC_TAG) + except process.HoldTagExistsError: + # We can ignore this error. + None + + if previous_snapshot: + s = new_snapshot.send(base=previous_snapshot["src"].__str__(), replicate=True) + else: + s = new_snapshot.send(replicate=True) + + try: + r = zfs.receive(dest.path(), force=True) + shutil.copyfileobj(s, r) + + s.close() + r.close() + except: + try: + s.close() + except: + None + + try: + r.close() + except: + None + + new_snapshot.release(HOLD_SYNC_TAG) + new_snapshot.destroy() + return False + + # Hold the dest tag + dest_snapshot = dest.snapshot(new_snapshot.snapname(), create=False) + dest_snapshot.hold(HOLD_SYNC_TAG) + + # Release the hold + if previous_snapshot: + for _, snap in previous_snapshot.items(): + snap.release(HOLD_SYNC_TAG) + snap.destroy() + + return True + +def main(): + if os.geteuid(): + print("zfssync must be run as root") + sys.exit(1) + + # Configure logging + setup_logging() + + if len(sys.argv) != 2: + logging.error("Project argument is required") + sys.exit(2) + + # Setup zfssync + logging.info('Starting zfssync') + + with open('/home/mirror/merlin/zfssync.yml') as f: + conf = yaml.load(f) + + project_name = sys.argv[1] + if project_name in conf['projects']: + project = conf['projects'][project_name] + else: + print('No project "{}" in config'.format(project_name)) + sys.exit(3) + + hostname = socket.gethostname() + dest_hostname = 'phys-1002-201.cloud.cs.uwaterloo.ca' if hostname == 'potassium-benzoate' else 'potassium-benzoate' + + logging.info('Sending {} from {} to {}'.format(project_name, hostname, dest_hostname)) + + src = Project( + project['hosts'][hostname]['pool'], + project['hosts'][hostname]['dataset']) + dest = Project( + project['hosts'][dest_hostname]['pool'], + project['hosts'][dest_hostname]['dataset'], + host=dest_hostname) + + sync(src, dest) + +if __name__ == '__main__': + main() diff --git a/roles/nginx/tasks/main.yml b/roles/nginx/tasks/main.yml index 25a8372..109eb1a 100644 --- a/roles/nginx/tasks/main.yml +++ b/roles/nginx/tasks/main.yml @@ -41,7 +41,7 @@ - name: enable debian.conf and mirror.conf # noqa deprecated-command-syntax - command: + command: cmd: "ln -s ../sites-available/{{ item }} {{ item }}" chdir: "/etc/nginx/sites-enabled" creates: "/etc/nginx/sites-enabled/{{ item }}" diff --git a/roles/ssh/tasks/main.yml b/roles/ssh/tasks/main.yml index 49fb781..73aff77 100644 --- a/roles/ssh/tasks/main.yml +++ b/roles/ssh/tasks/main.yml @@ -19,10 +19,10 @@ owner: root group: root mode: "0644" - + - name: generate keys for sshd # is ssh_host_dsa_key or ssh_host_ecdsa_key used anywhere? - command: + command: cmd: > ssh-keygen -q -t {{ item.type }} diff --git a/roles/system/tasks/main.yml b/roles/system/tasks/main.yml index c6b0dc3..0671f79 100755 --- a/roles/system/tasks/main.yml +++ b/roles/system/tasks/main.yml @@ -76,9 +76,9 @@ # create the var using # vars: # disks: -# - /dev/vdc -# - /dev/vdd -# - /dev/vde +# - /dev/vdc +# - /dev/vdd +# - /dev/vde # - /dev/vdf # disk_arg: "{{ disks | join(' ') }}" @@ -87,9 +87,9 @@ - name: disks to be used for zpool set_fact: disks: - - /dev/vdc - - /dev/vdd - - /dev/vde + - /dev/vdc + - /dev/vdd + - /dev/vde - /dev/vdf - name: join disk pathes onto one line @@ -104,9 +104,9 @@ - name: create and mount zpool command: > - zpool create - -m /mirror/root/.cscmirror - cscmirror - raidz2 + zpool create + -m /mirror/root/.cscmirror + cscmirror + raidz2 {{ disk_arg }} when: zpool_exists.rc != 0