Merge branch 'master' of https://git.csclub.uwaterloo.ca/public/mirror-env
commit
be4b924197
@ -0,0 +1,152 @@ |
||||
#!/usr/bin/env python3 |
||||
|
||||
import logging, sys, os, shutil, time, datetime, yaml, socket |
||||
from weir import zfs, process |
||||
|
||||
HOLD_SYNC_TAG = 'sync' |
||||
|
||||
def setup_logging(): |
||||
logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s") |
||||
|
||||
class Project: |
||||
def __init__(self, pool, name, host=None): |
||||
self.host = host |
||||
self.pool = pool |
||||
self.name = name |
||||
|
||||
self.fs = zfs.open(self.path(), 'filesystem') |
||||
|
||||
# Check that the filesystem exists |
||||
try: |
||||
self.fs.getprops() |
||||
except process.DatasetNotFoundError: |
||||
self.fs = zfs.create(self.path(), 'filesystem') |
||||
|
||||
|
||||
def path(self): |
||||
if self.host: |
||||
return 'zfs://{}/{}/{}'.format(self.host, self.pool, self.name) |
||||
|
||||
return '{}/{}'.format(self.pool, self.name) |
||||
|
||||
def snapshots(self): |
||||
return self.fs.snapshots() |
||||
|
||||
def snapshot(self, name=None, create=True): |
||||
if name is None: |
||||
name = 'test' |
||||
|
||||
snap = list(filter(lambda s: s.snapname() == name, self.snapshots())) |
||||
if len(snap) == 0 and create: |
||||
return self.fs.snapshot(name) |
||||
elif len(snap) == 0: |
||||
return None |
||||
|
||||
return snap[0] |
||||
|
||||
|
||||
|
||||
def sync(src, dest): |
||||
# Find the last snapshot in common between src and dest |
||||
src_snapshots = map(lambda s: s.snapname(), src.snapshots()) |
||||
dest_snapshots = map(lambda s: s.snapname(), dest.snapshots()) |
||||
common_snapshots = list(set(src_snapshots) & set(dest_snapshots)) |
||||
common_snapshots.sort() |
||||
|
||||
previous_snapshot = None |
||||
if len(common_snapshots): |
||||
previous_snapshot = dict([ |
||||
("src", src.snapshot(common_snapshots[-1], create=False)), |
||||
("dest", dest.snapshot(common_snapshots[-1], create=False)) |
||||
]) |
||||
|
||||
# Create the new snapshot |
||||
new_snapshot = src.snapshot(datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")) |
||||
|
||||
# Hold the source snapshot |
||||
try: |
||||
new_snapshot.hold(HOLD_SYNC_TAG) |
||||
except process.HoldTagExistsError: |
||||
# We can ignore this error. |
||||
None |
||||
|
||||
if previous_snapshot: |
||||
s = new_snapshot.send(base=previous_snapshot["src"].__str__(), replicate=True) |
||||
else: |
||||
s = new_snapshot.send(replicate=True) |
||||
|
||||
try: |
||||
r = zfs.receive(dest.path(), force=True) |
||||
shutil.copyfileobj(s, r) |
||||
|
||||
s.close() |
||||
r.close() |
||||
except: |
||||
try: |
||||
s.close() |
||||
except: |
||||
None |
||||
|
||||
try: |
||||
r.close() |
||||
except: |
||||
None |
||||
|
||||
new_snapshot.release(HOLD_SYNC_TAG) |
||||
new_snapshot.destroy() |
||||
return False |
||||
|
||||
# Hold the dest tag |
||||
dest_snapshot = dest.snapshot(new_snapshot.snapname(), create=False) |
||||
dest_snapshot.hold(HOLD_SYNC_TAG) |
||||
|
||||
# Release the hold |
||||
if previous_snapshot: |
||||
for _, snap in previous_snapshot.items(): |
||||
snap.release(HOLD_SYNC_TAG) |
||||
snap.destroy() |
||||
|
||||
return True |
||||
|
||||
def main(): |
||||
if os.geteuid(): |
||||
print("zfssync must be run as root") |
||||
sys.exit(1) |
||||
|
||||
# Configure logging |
||||
setup_logging() |
||||
|
||||
if len(sys.argv) != 2: |
||||
logging.error("Project argument is required") |
||||
sys.exit(2) |
||||
|
||||
# Setup zfssync |
||||
logging.info('Starting zfssync') |
||||
|
||||
with open('/home/mirror/merlin/zfssync.yml') as f: |
||||
conf = yaml.load(f) |
||||
|
||||
project_name = sys.argv[1] |
||||
if project_name in conf['projects']: |
||||
project = conf['projects'][project_name] |
||||
else: |
||||
print('No project "{}" in config'.format(project_name)) |
||||
sys.exit(3) |
||||
|
||||
hostname = socket.gethostname() |
||||
dest_hostname = 'phys-1002-201.cloud.cs.uwaterloo.ca' if hostname == 'potassium-benzoate' else 'potassium-benzoate' |
||||
|
||||
logging.info('Sending {} from {} to {}'.format(project_name, hostname, dest_hostname)) |
||||
|
||||
src = Project( |
||||
project['hosts'][hostname]['pool'], |
||||
project['hosts'][hostname]['dataset']) |
||||
dest = Project( |
||||
project['hosts'][dest_hostname]['pool'], |
||||
project['hosts'][dest_hostname]['dataset'], |
||||
host=dest_hostname) |
||||
|
||||
sync(src, dest) |
||||
|
||||
if __name__ == '__main__': |
||||
main() |
Loading…
Reference in new issue