A dev environment for the mirror.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 
mirror-env/roles/mirror/templates/zfssync.py

152 lines
4.0 KiB

#!/usr/bin/env python3
import logging, sys, os, shutil, time, datetime, yaml, socket
from weir import zfs, process
HOLD_SYNC_TAG = 'sync'
def setup_logging():
logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s")
class Project:
def __init__(self, pool, name, host=None):
self.host = host
self.pool = pool
self.name = name
self.fs = zfs.open(self.path(), 'filesystem')
# Check that the filesystem exists
try:
self.fs.getprops()
except process.DatasetNotFoundError:
self.fs = zfs.create(self.path(), 'filesystem')
def path(self):
if self.host:
return 'zfs://{}/{}/{}'.format(self.host, self.pool, self.name)
return '{}/{}'.format(self.pool, self.name)
def snapshots(self):
return self.fs.snapshots()
def snapshot(self, name=None, create=True):
if name is None:
name = 'test'
snap = list(filter(lambda s: s.snapname() == name, self.snapshots()))
if len(snap) == 0 and create:
return self.fs.snapshot(name)
elif len(snap) == 0:
return None
return snap[0]
def sync(src, dest):
# Find the last snapshot in common between src and dest
src_snapshots = map(lambda s: s.snapname(), src.snapshots())
dest_snapshots = map(lambda s: s.snapname(), dest.snapshots())
common_snapshots = list(set(src_snapshots) & set(dest_snapshots))
common_snapshots.sort()
previous_snapshot = None
if len(common_snapshots):
previous_snapshot = dict([
("src", src.snapshot(common_snapshots[-1], create=False)),
("dest", dest.snapshot(common_snapshots[-1], create=False))
])
# Create the new snapshot
new_snapshot = src.snapshot(datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S"))
# Hold the source snapshot
try:
new_snapshot.hold(HOLD_SYNC_TAG)
except process.HoldTagExistsError:
# We can ignore this error.
None
if previous_snapshot:
s = new_snapshot.send(base=previous_snapshot["src"].__str__(), replicate=True)
else:
s = new_snapshot.send(replicate=True)
try:
r = zfs.receive(dest.path(), force=True)
shutil.copyfileobj(s, r)
s.close()
r.close()
except:
try:
s.close()
except:
None
try:
r.close()
except:
None
new_snapshot.release(HOLD_SYNC_TAG)
new_snapshot.destroy()
return False
# Hold the dest tag
dest_snapshot = dest.snapshot(new_snapshot.snapname(), create=False)
dest_snapshot.hold(HOLD_SYNC_TAG)
# Release the hold
if previous_snapshot:
for _, snap in previous_snapshot.items():
snap.release(HOLD_SYNC_TAG)
snap.destroy()
return True
def main():
if os.geteuid():
print("zfssync must be run as root")
sys.exit(1)
# Configure logging
setup_logging()
if len(sys.argv) != 2:
logging.error("Project argument is required")
sys.exit(2)
# Setup zfssync
logging.info('Starting zfssync')
with open('/home/mirror/merlin/zfssync.yml') as f:
conf = yaml.load(f)
project_name = sys.argv[1]
if project_name in conf['projects']:
project = conf['projects'][project_name]
else:
print('No project "{}" in config'.format(project_name))
sys.exit(3)
hostname = socket.gethostname()
dest_hostname = 'phys-1002-201.cloud.cs.uwaterloo.ca' if hostname == 'potassium-benzoate' else 'potassium-benzoate'
logging.info('Sending {} from {} to {}'.format(project_name, hostname, dest_hostname))
src = Project(
project['hosts'][hostname]['pool'],
project['hosts'][hostname]['dataset'])
dest = Project(
project['hosts'][dest_hostname]['pool'],
project['hosts'][dest_hostname]['dataset'],
host=dest_hostname)
sync(src, dest)
if __name__ == '__main__':
main()