Merge branch 'master' of https://git.csclub.uwaterloo.ca/public/mirror-env

roles/ftp/tasks/main.yml

@@ -30,9 +30,9 @@
   # will not regenerated every playbook run
   command:
     cmd: >
-      openssl dhparam 
-      -outform PEM -2|-5 
-      1024|1536|2048|3072|4096|6144|7680|8192 
+      openssl dhparam
+      -outform PEM -2|-5
+      1024|1536|2048|3072|4096|6144|7680|8192
       > /etc/proftpd/dhparams.pem
     creates: /etc/proftpd/dhparams.pem
 

roles/mirror/tasks/main.yml

@@ -17,6 +17,14 @@
     group: root
     mode: "0644"
 
+- name: copy zfssync python script
+  copy:
+    src: "{{ role_path }}/templates/zfssync.py"
+    dest: /usr/local/bin/zfssync.py
+    owner: root
+    group: root
+    mode: "0755"
+
 - name: restart and enable cron
   systemd:
     name: cron
@@ -65,7 +73,7 @@
 - name: create symlinks from busybox and arthur
   # noqa deprecated-command-syntax
   # need chdir to create relative symlinks
-  command: 
+  command:
     cmd: "ln -s {{ item.src }} {{ item.dest }}"
     chdir: "/mirror/merlin/bin"
     creates: "/mirror/merlin/bin/{{ item.dest }}"

roles/mirror/templates/zfssync.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+
+import logging, sys, os, shutil, time, datetime, yaml, socket
+from weir import zfs, process
+
+HOLD_SYNC_TAG = 'sync'
+
+def setup_logging():
+    logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s")
+
+class Project:
+    def __init__(self, pool, name, host=None):
+        self.host = host
+        self.pool = pool
+        self.name = name
+
+        self.fs = zfs.open(self.path(), 'filesystem')
+
+        # Check that the filesystem exists
+        try:
+            self.fs.getprops()
+        except process.DatasetNotFoundError:
+            self.fs = zfs.create(self.path(), 'filesystem')
+
+
+    def path(self):
+        if self.host:
+            return 'zfs://{}/{}/{}'.format(self.host, self.pool, self.name)
+
+        return '{}/{}'.format(self.pool, self.name)
+
+    def snapshots(self):
+        return self.fs.snapshots()
+
+    def snapshot(self, name=None, create=True):
+        if name is None:
+            name = 'test'
+
+        snap = list(filter(lambda s: s.snapname() == name, self.snapshots()))
+        if len(snap) == 0 and create:
+            return self.fs.snapshot(name)
+        elif len(snap) == 0:
+            return None
+
+        return snap[0]
+
+
+
+def sync(src, dest):
+    # Find the last snapshot in common between src and dest
+    src_snapshots = map(lambda s: s.snapname(), src.snapshots())
+    dest_snapshots = map(lambda s: s.snapname(), dest.snapshots())
+    common_snapshots = list(set(src_snapshots) & set(dest_snapshots))
+    common_snapshots.sort()
+
+    previous_snapshot = None
+    if len(common_snapshots):
+       previous_snapshot = dict([
+           ("src", src.snapshot(common_snapshots[-1], create=False)),
+           ("dest", dest.snapshot(common_snapshots[-1], create=False))
+       ])
+
+    # Create the new snapshot
+    new_snapshot = src.snapshot(datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S"))
+
+    # Hold the source snapshot
+    try:
+        new_snapshot.hold(HOLD_SYNC_TAG)
+    except process.HoldTagExistsError:
+        # We can ignore this error.
+        None
+
+    if previous_snapshot:
+        s = new_snapshot.send(base=previous_snapshot["src"].__str__(), replicate=True)
+    else:
+        s = new_snapshot.send(replicate=True)
+
+    try:
+        r = zfs.receive(dest.path(), force=True)
+        shutil.copyfileobj(s, r)
+
+        s.close()
+        r.close()
+    except:
+        try:
+            s.close()
+        except:
+            None
+
+        try:
+            r.close()
+        except:
+            None
+
+        new_snapshot.release(HOLD_SYNC_TAG)
+        new_snapshot.destroy()
+        return False
+
+    # Hold the dest tag
+    dest_snapshot = dest.snapshot(new_snapshot.snapname(), create=False)
+    dest_snapshot.hold(HOLD_SYNC_TAG)
+
+    # Release the hold
+    if previous_snapshot:
+        for _, snap in previous_snapshot.items():
+            snap.release(HOLD_SYNC_TAG)
+            snap.destroy()
+
+    return True
+
+def main():
+    if os.geteuid():
+        print("zfssync must be run as root")
+        sys.exit(1)
+
+    # Configure logging
+    setup_logging()
+
+    if len(sys.argv) != 2:
+        logging.error("Project argument is required")
+        sys.exit(2)
+
+    # Setup zfssync
+    logging.info('Starting zfssync')
+
+    with open('/home/mirror/merlin/zfssync.yml') as f:
+        conf = yaml.load(f)
+
+    project_name = sys.argv[1]
+    if project_name in conf['projects']:
+        project = conf['projects'][project_name]
+    else:
+        print('No project "{}" in config'.format(project_name))
+        sys.exit(3)
+
+    hostname = socket.gethostname()
+    dest_hostname = 'phys-1002-201.cloud.cs.uwaterloo.ca' if hostname == 'potassium-benzoate' else 'potassium-benzoate'
+
+    logging.info('Sending {} from {} to {}'.format(project_name, hostname, dest_hostname))
+
+    src = Project(
+            project['hosts'][hostname]['pool'],
+            project['hosts'][hostname]['dataset'])
+    dest = Project(
+            project['hosts'][dest_hostname]['pool'],
+            project['hosts'][dest_hostname]['dataset'],
+            host=dest_hostname)
+
+    sync(src, dest)
+
+if __name__ == '__main__':
+    main()

roles/nginx/tasks/main.yml

@@ -41,7 +41,7 @@
 
 - name: enable debian.conf and mirror.conf
   # noqa deprecated-command-syntax
-  command: 
+  command:
     cmd: "ln -s ../sites-available/{{ item }} {{ item }}"
     chdir: "/etc/nginx/sites-enabled"
     creates: "/etc/nginx/sites-enabled/{{ item }}"

roles/ssh/tasks/main.yml

@@ -19,10 +19,10 @@
     owner: root
     group: root
     mode: "0644"
-    
+
 - name: generate keys for sshd
   # is ssh_host_dsa_key or ssh_host_ecdsa_key used anywhere?
-  command: 
+  command:
     cmd: >
       ssh-keygen -q
       -t {{ item.type }}