add zfssync.py and remove trailing whitespace
This commit is contained in:
parent
81e49c21e9
commit
a1da002935
|
@ -30,9 +30,9 @@
|
||||||
# will not regenerated every playbook run
|
# will not regenerated every playbook run
|
||||||
command:
|
command:
|
||||||
cmd: >
|
cmd: >
|
||||||
openssl dhparam
|
openssl dhparam
|
||||||
-outform PEM -2|-5
|
-outform PEM -2|-5
|
||||||
1024|1536|2048|3072|4096|6144|7680|8192
|
1024|1536|2048|3072|4096|6144|7680|8192
|
||||||
> /etc/proftpd/dhparams.pem
|
> /etc/proftpd/dhparams.pem
|
||||||
creates: /etc/proftpd/dhparams.pem
|
creates: /etc/proftpd/dhparams.pem
|
||||||
|
|
||||||
|
|
|
@ -6,6 +6,14 @@
|
||||||
group: root
|
group: root
|
||||||
mode: "0644"
|
mode: "0644"
|
||||||
|
|
||||||
|
- name: copy zfssync python script
|
||||||
|
copy:
|
||||||
|
src: "{{ role_path }}/templates/zfssync.py"
|
||||||
|
dest: /usr/local/bin/zfssync.py
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0755"
|
||||||
|
|
||||||
- name: restart and enable cron
|
- name: restart and enable cron
|
||||||
systemd:
|
systemd:
|
||||||
name: cron
|
name: cron
|
||||||
|
@ -21,7 +29,7 @@
|
||||||
mode: preserve
|
mode: preserve
|
||||||
|
|
||||||
- name: copy files for /mirror/root
|
- name: copy files for /mirror/root
|
||||||
copy:
|
copy:
|
||||||
src: "{{ role_path }}/templates/root/root/"
|
src: "{{ role_path }}/templates/root/root/"
|
||||||
dest: /mirror/root
|
dest: /mirror/root
|
||||||
owner: root
|
owner: root
|
||||||
|
@ -54,7 +62,7 @@
|
||||||
- name: create symlinks from busybox and arthur
|
- name: create symlinks from busybox and arthur
|
||||||
# noqa deprecated-command-syntax
|
# noqa deprecated-command-syntax
|
||||||
# need chdir to create relative symlinks
|
# need chdir to create relative symlinks
|
||||||
command:
|
command:
|
||||||
cmd: "ln -s {{ item.src }} {{ item.dest }}"
|
cmd: "ln -s {{ item.src }} {{ item.dest }}"
|
||||||
chdir: "/mirror/merlin/bin"
|
chdir: "/mirror/merlin/bin"
|
||||||
creates: "/mirror/merlin/bin/{{ item.dest }}"
|
creates: "/mirror/merlin/bin/{{ item.dest }}"
|
||||||
|
@ -84,7 +92,7 @@
|
||||||
- name: create first merlin.sock symlink
|
- name: create first merlin.sock symlink
|
||||||
# noqa deprecated-command-syntax
|
# noqa deprecated-command-syntax
|
||||||
# need chdir to create relative symlinks
|
# need chdir to create relative symlinks
|
||||||
command:
|
command:
|
||||||
cmd: "ln -s run/merlin.sock merlin.sock"
|
cmd: "ln -s run/merlin.sock merlin.sock"
|
||||||
chdir: "/mirror/merlin"
|
chdir: "/mirror/merlin"
|
||||||
creates: "/mirror/merlin/merlin.sock"
|
creates: "/mirror/merlin/merlin.sock"
|
||||||
|
|
|
@ -0,0 +1,152 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import logging, sys, os, shutil, time, datetime, yaml, socket
|
||||||
|
from weir import zfs, process
|
||||||
|
|
||||||
|
HOLD_SYNC_TAG = 'sync'
|
||||||
|
|
||||||
|
def setup_logging():
|
||||||
|
logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s")
|
||||||
|
|
||||||
|
class Project:
|
||||||
|
def __init__(self, pool, name, host=None):
|
||||||
|
self.host = host
|
||||||
|
self.pool = pool
|
||||||
|
self.name = name
|
||||||
|
|
||||||
|
self.fs = zfs.open(self.path(), 'filesystem')
|
||||||
|
|
||||||
|
# Check that the filesystem exists
|
||||||
|
try:
|
||||||
|
self.fs.getprops()
|
||||||
|
except process.DatasetNotFoundError:
|
||||||
|
self.fs = zfs.create(self.path(), 'filesystem')
|
||||||
|
|
||||||
|
|
||||||
|
def path(self):
|
||||||
|
if self.host:
|
||||||
|
return 'zfs://{}/{}/{}'.format(self.host, self.pool, self.name)
|
||||||
|
|
||||||
|
return '{}/{}'.format(self.pool, self.name)
|
||||||
|
|
||||||
|
def snapshots(self):
|
||||||
|
return self.fs.snapshots()
|
||||||
|
|
||||||
|
def snapshot(self, name=None, create=True):
|
||||||
|
if name is None:
|
||||||
|
name = 'test'
|
||||||
|
|
||||||
|
snap = list(filter(lambda s: s.snapname() == name, self.snapshots()))
|
||||||
|
if len(snap) == 0 and create:
|
||||||
|
return self.fs.snapshot(name)
|
||||||
|
elif len(snap) == 0:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return snap[0]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def sync(src, dest):
|
||||||
|
# Find the last snapshot in common between src and dest
|
||||||
|
src_snapshots = map(lambda s: s.snapname(), src.snapshots())
|
||||||
|
dest_snapshots = map(lambda s: s.snapname(), dest.snapshots())
|
||||||
|
common_snapshots = list(set(src_snapshots) & set(dest_snapshots))
|
||||||
|
common_snapshots.sort()
|
||||||
|
|
||||||
|
previous_snapshot = None
|
||||||
|
if len(common_snapshots):
|
||||||
|
previous_snapshot = dict([
|
||||||
|
("src", src.snapshot(common_snapshots[-1], create=False)),
|
||||||
|
("dest", dest.snapshot(common_snapshots[-1], create=False))
|
||||||
|
])
|
||||||
|
|
||||||
|
# Create the new snapshot
|
||||||
|
new_snapshot = src.snapshot(datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S"))
|
||||||
|
|
||||||
|
# Hold the source snapshot
|
||||||
|
try:
|
||||||
|
new_snapshot.hold(HOLD_SYNC_TAG)
|
||||||
|
except process.HoldTagExistsError:
|
||||||
|
# We can ignore this error.
|
||||||
|
None
|
||||||
|
|
||||||
|
if previous_snapshot:
|
||||||
|
s = new_snapshot.send(base=previous_snapshot["src"].__str__(), replicate=True)
|
||||||
|
else:
|
||||||
|
s = new_snapshot.send(replicate=True)
|
||||||
|
|
||||||
|
try:
|
||||||
|
r = zfs.receive(dest.path(), force=True)
|
||||||
|
shutil.copyfileobj(s, r)
|
||||||
|
|
||||||
|
s.close()
|
||||||
|
r.close()
|
||||||
|
except:
|
||||||
|
try:
|
||||||
|
s.close()
|
||||||
|
except:
|
||||||
|
None
|
||||||
|
|
||||||
|
try:
|
||||||
|
r.close()
|
||||||
|
except:
|
||||||
|
None
|
||||||
|
|
||||||
|
new_snapshot.release(HOLD_SYNC_TAG)
|
||||||
|
new_snapshot.destroy()
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Hold the dest tag
|
||||||
|
dest_snapshot = dest.snapshot(new_snapshot.snapname(), create=False)
|
||||||
|
dest_snapshot.hold(HOLD_SYNC_TAG)
|
||||||
|
|
||||||
|
# Release the hold
|
||||||
|
if previous_snapshot:
|
||||||
|
for _, snap in previous_snapshot.items():
|
||||||
|
snap.release(HOLD_SYNC_TAG)
|
||||||
|
snap.destroy()
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if os.geteuid():
|
||||||
|
print("zfssync must be run as root")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Configure logging
|
||||||
|
setup_logging()
|
||||||
|
|
||||||
|
if len(sys.argv) != 2:
|
||||||
|
logging.error("Project argument is required")
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
# Setup zfssync
|
||||||
|
logging.info('Starting zfssync')
|
||||||
|
|
||||||
|
with open('/home/mirror/merlin/zfssync.yml') as f:
|
||||||
|
conf = yaml.load(f)
|
||||||
|
|
||||||
|
project_name = sys.argv[1]
|
||||||
|
if project_name in conf['projects']:
|
||||||
|
project = conf['projects'][project_name]
|
||||||
|
else:
|
||||||
|
print('No project "{}" in config'.format(project_name))
|
||||||
|
sys.exit(3)
|
||||||
|
|
||||||
|
hostname = socket.gethostname()
|
||||||
|
dest_hostname = 'phys-1002-201.cloud.cs.uwaterloo.ca' if hostname == 'potassium-benzoate' else 'potassium-benzoate'
|
||||||
|
|
||||||
|
logging.info('Sending {} from {} to {}'.format(project_name, hostname, dest_hostname))
|
||||||
|
|
||||||
|
src = Project(
|
||||||
|
project['hosts'][hostname]['pool'],
|
||||||
|
project['hosts'][hostname]['dataset'])
|
||||||
|
dest = Project(
|
||||||
|
project['hosts'][dest_hostname]['pool'],
|
||||||
|
project['hosts'][dest_hostname]['dataset'],
|
||||||
|
host=dest_hostname)
|
||||||
|
|
||||||
|
sync(src, dest)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
|
@ -41,7 +41,7 @@
|
||||||
|
|
||||||
- name: enable debian.conf and mirror.conf
|
- name: enable debian.conf and mirror.conf
|
||||||
# noqa deprecated-command-syntax
|
# noqa deprecated-command-syntax
|
||||||
command:
|
command:
|
||||||
cmd: "ln -s ../sites-available/{{ item }} {{ item }}"
|
cmd: "ln -s ../sites-available/{{ item }} {{ item }}"
|
||||||
chdir: "/etc/nginx/sites-enabled"
|
chdir: "/etc/nginx/sites-enabled"
|
||||||
creates: "/etc/nginx/sites-enabled/{{ item }}"
|
creates: "/etc/nginx/sites-enabled/{{ item }}"
|
||||||
|
|
|
@ -19,10 +19,10 @@
|
||||||
owner: root
|
owner: root
|
||||||
group: root
|
group: root
|
||||||
mode: "0644"
|
mode: "0644"
|
||||||
|
|
||||||
- name: generate keys for sshd
|
- name: generate keys for sshd
|
||||||
# is ssh_host_dsa_key or ssh_host_ecdsa_key used anywhere?
|
# is ssh_host_dsa_key or ssh_host_ecdsa_key used anywhere?
|
||||||
command:
|
command:
|
||||||
cmd: >
|
cmd: >
|
||||||
ssh-keygen -q
|
ssh-keygen -q
|
||||||
-t {{ item.type }}
|
-t {{ item.type }}
|
||||||
|
|
|
@ -76,9 +76,9 @@
|
||||||
# create the var using
|
# create the var using
|
||||||
# vars:
|
# vars:
|
||||||
# disks:
|
# disks:
|
||||||
# - /dev/vdc
|
# - /dev/vdc
|
||||||
# - /dev/vdd
|
# - /dev/vdd
|
||||||
# - /dev/vde
|
# - /dev/vde
|
||||||
# - /dev/vdf
|
# - /dev/vdf
|
||||||
# disk_arg: "{{ disks | join(' ') }}"
|
# disk_arg: "{{ disks | join(' ') }}"
|
||||||
|
|
||||||
|
@ -87,9 +87,9 @@
|
||||||
- name: disks to be used for zpool
|
- name: disks to be used for zpool
|
||||||
set_fact:
|
set_fact:
|
||||||
disks:
|
disks:
|
||||||
- /dev/vdc
|
- /dev/vdc
|
||||||
- /dev/vdd
|
- /dev/vdd
|
||||||
- /dev/vde
|
- /dev/vde
|
||||||
- /dev/vdf
|
- /dev/vdf
|
||||||
|
|
||||||
- name: join disk pathes onto one line
|
- name: join disk pathes onto one line
|
||||||
|
@ -104,9 +104,9 @@
|
||||||
|
|
||||||
- name: create and mount zpool
|
- name: create and mount zpool
|
||||||
command: >
|
command: >
|
||||||
zpool create
|
zpool create
|
||||||
-m /mirror/root/.cscmirror
|
-m /mirror/root/.cscmirror
|
||||||
cscmirror
|
cscmirror
|
||||||
raidz2
|
raidz2
|
||||||
{{ disk_arg }}
|
{{ disk_arg }}
|
||||||
when: zpool_exists.rc != 0
|
when: zpool_exists.rc != 0
|
||||||
|
|
Loading…
Reference in New Issue