diff --git a/roles/ftp/tasks/main.yml b/roles/ftp/tasks/main.yml index 45f32e6..7a94a0f 100644 --- a/roles/ftp/tasks/main.yml +++ b/roles/ftp/tasks/main.yml @@ -30,9 +30,9 @@ # will not regenerated every playbook run command: cmd: > - openssl dhparam - -outform PEM -2|-5 - 1024|1536|2048|3072|4096|6144|7680|8192 + openssl dhparam + -outform PEM -2|-5 + 1024|1536|2048|3072|4096|6144|7680|8192 > /etc/proftpd/dhparams.pem creates: /etc/proftpd/dhparams.pem diff --git a/roles/mirror/tasks/main.yml b/roles/mirror/tasks/main.yml index 81fc636..97e3695 100644 --- a/roles/mirror/tasks/main.yml +++ b/roles/mirror/tasks/main.yml @@ -17,6 +17,14 @@ group: root mode: "0644" +- name: copy zfssync python script + copy: + src: "{{ role_path }}/templates/zfssync.py" + dest: /usr/local/bin/zfssync.py + owner: root + group: root + mode: "0755" + - name: restart and enable cron systemd: name: cron @@ -65,7 +73,7 @@ - name: create symlinks from busybox and arthur # noqa deprecated-command-syntax # need chdir to create relative symlinks - command: + command: cmd: "ln -s {{ item.src }} {{ item.dest }}" chdir: "/mirror/merlin/bin" creates: "/mirror/merlin/bin/{{ item.dest }}" diff --git a/roles/mirror/templates/zfssync.py b/roles/mirror/templates/zfssync.py new file mode 100755 index 0000000..3c2a0e7 --- /dev/null +++ b/roles/mirror/templates/zfssync.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 + +import logging, sys, os, shutil, time, datetime, yaml, socket +from weir import zfs, process + +HOLD_SYNC_TAG = 'sync' + +def setup_logging(): + logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s") + +class Project: + def __init__(self, pool, name, host=None): + self.host = host + self.pool = pool + self.name = name + + self.fs = zfs.open(self.path(), 'filesystem') + + # Check that the filesystem exists + try: + self.fs.getprops() + except process.DatasetNotFoundError: + self.fs = zfs.create(self.path(), 'filesystem') + + + def path(self): + if self.host: + return 'zfs://{}/{}/{}'.format(self.host, self.pool, self.name) + + return '{}/{}'.format(self.pool, self.name) + + def snapshots(self): + return self.fs.snapshots() + + def snapshot(self, name=None, create=True): + if name is None: + name = 'test' + + snap = list(filter(lambda s: s.snapname() == name, self.snapshots())) + if len(snap) == 0 and create: + return self.fs.snapshot(name) + elif len(snap) == 0: + return None + + return snap[0] + + + +def sync(src, dest): + # Find the last snapshot in common between src and dest + src_snapshots = map(lambda s: s.snapname(), src.snapshots()) + dest_snapshots = map(lambda s: s.snapname(), dest.snapshots()) + common_snapshots = list(set(src_snapshots) & set(dest_snapshots)) + common_snapshots.sort() + + previous_snapshot = None + if len(common_snapshots): + previous_snapshot = dict([ + ("src", src.snapshot(common_snapshots[-1], create=False)), + ("dest", dest.snapshot(common_snapshots[-1], create=False)) + ]) + + # Create the new snapshot + new_snapshot = src.snapshot(datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")) + + # Hold the source snapshot + try: + new_snapshot.hold(HOLD_SYNC_TAG) + except process.HoldTagExistsError: + # We can ignore this error. + None + + if previous_snapshot: + s = new_snapshot.send(base=previous_snapshot["src"].__str__(), replicate=True) + else: + s = new_snapshot.send(replicate=True) + + try: + r = zfs.receive(dest.path(), force=True) + shutil.copyfileobj(s, r) + + s.close() + r.close() + except: + try: + s.close() + except: + None + + try: + r.close() + except: + None + + new_snapshot.release(HOLD_SYNC_TAG) + new_snapshot.destroy() + return False + + # Hold the dest tag + dest_snapshot = dest.snapshot(new_snapshot.snapname(), create=False) + dest_snapshot.hold(HOLD_SYNC_TAG) + + # Release the hold + if previous_snapshot: + for _, snap in previous_snapshot.items(): + snap.release(HOLD_SYNC_TAG) + snap.destroy() + + return True + +def main(): + if os.geteuid(): + print("zfssync must be run as root") + sys.exit(1) + + # Configure logging + setup_logging() + + if len(sys.argv) != 2: + logging.error("Project argument is required") + sys.exit(2) + + # Setup zfssync + logging.info('Starting zfssync') + + with open('/home/mirror/merlin/zfssync.yml') as f: + conf = yaml.load(f) + + project_name = sys.argv[1] + if project_name in conf['projects']: + project = conf['projects'][project_name] + else: + print('No project "{}" in config'.format(project_name)) + sys.exit(3) + + hostname = socket.gethostname() + dest_hostname = 'phys-1002-201.cloud.cs.uwaterloo.ca' if hostname == 'potassium-benzoate' else 'potassium-benzoate' + + logging.info('Sending {} from {} to {}'.format(project_name, hostname, dest_hostname)) + + src = Project( + project['hosts'][hostname]['pool'], + project['hosts'][hostname]['dataset']) + dest = Project( + project['hosts'][dest_hostname]['pool'], + project['hosts'][dest_hostname]['dataset'], + host=dest_hostname) + + sync(src, dest) + +if __name__ == '__main__': + main() diff --git a/roles/nginx/tasks/main.yml b/roles/nginx/tasks/main.yml index 25a8372..109eb1a 100644 --- a/roles/nginx/tasks/main.yml +++ b/roles/nginx/tasks/main.yml @@ -41,7 +41,7 @@ - name: enable debian.conf and mirror.conf # noqa deprecated-command-syntax - command: + command: cmd: "ln -s ../sites-available/{{ item }} {{ item }}" chdir: "/etc/nginx/sites-enabled" creates: "/etc/nginx/sites-enabled/{{ item }}" diff --git a/roles/ssh/tasks/main.yml b/roles/ssh/tasks/main.yml index 49fb781..73aff77 100644 --- a/roles/ssh/tasks/main.yml +++ b/roles/ssh/tasks/main.yml @@ -19,10 +19,10 @@ owner: root group: root mode: "0644" - + - name: generate keys for sshd # is ssh_host_dsa_key or ssh_host_ecdsa_key used anywhere? - command: + command: cmd: > ssh-keygen -q -t {{ item.type }}