mirror/merlin_old/merlin.py

729 lines
26 KiB
Python
Raw Normal View History

2021-09-09 00:31:28 -04:00
#!/usr/bin/python2
import time, sys, os, errno, logging, signal, copy, select, socket, grp, random
2021-09-09 00:31:28 -04:00
daily = 86400
twice_daily = 86400 / 2
hourly = 3600
bi_hourly = 7200
tri_hourly = 10800
twice_hourly = 1800
ten_minutely = 600
five_minutely = 300
maxtime = 86400
mintime = 60
jobs = {}
MAX_JOBS = 6
#earPath = '/home/mirror/merlin/merlin.sock'
earPath = '/mirror/merlin/run/merlin.sock'
cmd_buf_size = 4096
repos = {
'debian': {
'command': '~/bin/csc-sync-debian debian debian.mirror.rafal.ca debian',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
# 'debian-cdimage': {
# 'command': '~/bin/csc-sync-cdimage debian-cdimage cdimage.debian.org cdimage',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
# },
'ubuntu': {
'command': '~/bin/csc-sync-debian ubuntu archive.ubuntu.com ubuntu drescher.canonical.com',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'ubuntu-ports': {
'command': '~/bin/csc-sync-debian ubuntu-ports ports.ubuntu.com ubuntu-ports drescher.canonical.com',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'linuxmint-packages': {
'command': '~/bin/csc-sync-debian linuxmint-packages rsync-packages.linuxmint.com packages',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'debian-multimedia': {
'command': '~/bin/csc-sync-debian debian-multimedia www.deb-multimedia.org deb',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'debian-backports': {
'command': '~/bin/csc-sync-debian debian-backports debian.mirror.rafal.ca debian-backports',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
# 'debian-volatile': {
# 'command': '~/bin/csc-sync-debian debian-volatile debian.mirror.rafal.ca debian-volatile',
# 'interval': bi_hourly,
# 'max-sync-time': maxtime,
# },
'debian-security': {
'command': '~/bin/csc-sync-debian debian-security rsync.security.debian.org debian-security security-master.debian.org',
'interval': twice_hourly,
'max-sync-time': maxtime,
},
'ubuntu-releases': {
'command': '~/bin/csc-sync-standard ubuntu-releases rsync.releases.ubuntu.com releases',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'xubuntu-releases': {
'command': '~/bin/csc-sync-standard xubuntu-releases cdimage.ubuntu.com cdimage/xubuntu/releases/',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
# 'emdebian': {
# 'command': '~/bin/csc-sync-badperms emdebian www.emdebian.org debian',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
# },
'puppylinux': {
'command': '~/bin/csc-sync-standard puppylinux distro.ibiblio.org puppylinux',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'CPAN': {
'command': '~/bin/csc-sync-standard CPAN cpan-rsync.perl.org CPAN',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'CRAN': {
'command': '~/bin/csc-sync-ssh CRAN cran.r-project.org "" cran-rsync ~/.ssh/id_cran_rsa',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'CTAN': {
'command': '~/bin/csc-sync-standard CTAN rsync.dante.ctan.org CTAN',
'interval': twice_daily,
'max-sync-time': maxtime,
},
# 'openoffice': {
# 'command': '~/bin/csc-sync-standard openoffice rsync.services.openoffice.org openoffice-extended',
# 'command': '~/bin/csc-sync-standard openoffice ftp.snt.utwente.nl openoffice-extended',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
# },
'fedora-epel': {
'command': '~/bin/csc-sync-standard fedora/epel mirrors.kernel.org fedora-epel && ~/bin/report_mirror >/dev/null',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'cygwin': {
'command': '~/bin/csc-sync-standard cygwin cygwin.com cygwin-ftp',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'gnu': {
#'command': '~/bin/csc-sync-standard gnu mirrors.ibiblio.org gnuftp/gnu/',
'command': '~/bin/csc-sync-standard gnu ftp.gnu.org gnu',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'nongnu': {
# 'command': '~/bin/csc-sync-standard nongnu dl.sv.gnu.org releases --ignore-errors',
'command': '~/bin/csc-sync-standard nongnu dl.sv.gnu.org releases',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'mysql': {
#'command': '~/bin/csc-sync-standard mysql mysql.he.net mysql',
'command': '~/bin/csc-sync-standard mysql rsync.mirrorservice.org ftp.mysql.com',
'interval': twice_daily,
'max-sync-time': maxtime,
},
# No longer syncs, and no longer really relevant
# 'mozdev': {
# 'command': '~/bin/csc-sync-standard mozdev rsync.mozdev.org mozdev',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
# },
'gnome': {
'command': '~/bin/csc-sync-standard gnome master.gnome.org gnomeftp gnome',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'damnsmalllinux': {
'command': '~/bin/csc-sync-standard damnsmalllinux ftp.heanet.ie mirrors/damnsmalllinux.org/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'linuxmint': {
'command': '~/bin/csc-sync-standard linuxmint pub.linuxmint.com pub',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'kernel.org-linux': {
'command': '~/bin/csc-sync-standard kernel.org/linux rsync.kernel.org pub/linux/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'kernel.org-software': {
'command': '~/bin/csc-sync-standard kernel.org/software rsync.kernel.org pub/software/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'apache': {
'command': '~/bin/csc-sync-apache apache rsync.us.apache.org apache-dist',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'eclipse': {
'command': '~/bin/csc-sync-standard eclipse download.eclipse.org eclipseMirror',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'kde': {
'command': '~/bin/csc-sync-standard kde rsync.kde.org kdeftp',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'kde-applicationdata': {
'command': '~/bin/csc-sync-standard kde-applicationdata rsync.kde.org applicationdata',
'interval': twice_daily,
'max-sync-time': maxtime,
},
# We are a Tier 1 arch mirror (https://bugs.archlinux.org/task/52853)
# so our IP is important.
'archlinux': {
#'command': '~/bin/csc-sync-standard archlinux archlinux.mirror.rafal.ca archlinux',
'command': '~/bin/csc-sync-archlinux archlinux',
'interval': five_minutely,
'max-sync-time': maxtime,
},
2021-11-18 02:50:39 -05:00
'artixlinux': {
'command': '~/bin/csc-sync-standard artixlinux mirror1.artixlinux.org artix-linux',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
2021-12-22 12:33:20 -05:00
# We stopped mirroring debian-ports on 2021-12-22 due to disk performance issues.
#'debian-ports': {
# 'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
#},
2021-09-09 00:31:28 -04:00
'slackware': {
'command': '~/bin/csc-sync-standard slackware slackware.cs.utah.edu slackware',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'debian-cd': {
'command': '~/bin/csc-sync-debian-cd',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'x.org': {
#'command': '~/bin/csc-sync-standard x.org xorg.freedesktop.org xorg-archive',
#'command': '~/bin/csc-sync-standard x.org mirror.us.leaseweb.net xorg',
'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub/',
2021-09-09 00:31:28 -04:00
'interval': twice_daily,
'max-sync-time': maxtime,
},
'centos': {
'command': '~/bin/csc-sync-standard centos us-msync.centos.org CentOS',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'opensuse': {
'command': '~/bin/csc-sync-standard opensuse stage.opensuse.org opensuse-full/opensuse/ #"--exclude distribution/.timestamp_invisible"',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'FreeBSD': {
# Has not updated since at least June 2018
#'command': '~/bin/csc-sync-standard FreeBSD ftp10.us.freebsd.org FreeBSD',
'command': '~/bin/csc-sync-standard FreeBSD ftp2.uk.freebsd.org ftp.freebsd.org/pub/FreeBSD/',
#'command': '~/bin/csc-sync-standard FreeBSD ftp3.us.freebsd.org FreeBSD/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'fedora-enchilada': {
# 'command': '~/bin/csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null',
'command': '~/bin/csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ && ~/bin/report_mirror >/dev/null',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'ubuntu-ports-releases': {
'command': '~/bin/csc-sync-standard ubuntu-ports-releases cdimage.ubuntu.com cdimage/releases/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'gentoo-distfiles': {
'command': '~/bin/csc-sync-gentoo',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'gentoo-portage': {
'command': '~/bin/csc-sync-standard gentoo-portage rsync1.us.gentoo.org gentoo-portage',
'interval': twice_hourly,
'max-sync-time': maxtime,
},
# This project is no longer available for mirroring
# https://bugzilla.mozilla.org/show_bug.cgi?id=807543
#'mozilla.org': {
# 'command': '~/bin/csc-sync-standard mozilla.org releases-rsync.mozilla.org mozilla-releases',
# 'interval': twice_hourly,
# 'max-sync-time': maxtime,
#},
'gutenberg': {
'command': '~/bin/csc-sync-standard gutenberg ftp@ftp.ibiblio.org gutenberg',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'racket-installers': {
'command': '~/bin/csc-sync-wget racket/racket-installers https://mirror.racket-lang.org/installers/ 1',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'plt-bundles': {
'command': '~/bin/csc-sync-standard racket/plt-bundles mirror.racket-lang.org plt-bundles',
'interval': twice_daily,
'max-sync-time': maxtime,
},
2021-12-22 12:21:37 -05:00
# We stopped mirroring OpenBSD on 2021-12-22 due to degraded disk performance
#'OpenBSD': {
# 'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
#},
2021-09-09 00:31:28 -04:00
'xiph': {
#'command': '~/bin/csc-sync-standard xiph downloads.xiph.org xiph/releases',
'command': '~/bin/csc-sync-standard xiph ftp.osuosl.org xiph',
'interval': twice_daily,
'max-sync-time': maxtime,
},
# We currently don't have the disk space
'netbsd': {
'command': '~/bin/csc-sync-standard NetBSD rsync.netbsd.org NetBSD',
'interval': twice_daily,
'max-sync-time': maxtime,
},
2021-12-22 13:07:02 -05:00
# We stopped mirroring pkgsrc on 2021-12-22 due to disk performance issues.
#'netbsd-pkgsrc': {
# 'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc',
# #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
#},
2021-09-09 00:31:28 -04:00
'macports-release': {
'command': '~/bin/csc-sync-standard MacPorts/release rsync.macports.org macports/release/',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'macports-distfiles': {
'command': '~/bin/csc-sync-standard MacPorts/mpdistfiles rsync.macports.org macports/distfiles/',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
# 'raspberrypi': {
# 'command': '~/bin/csc-sync-standard raspberrypi mirrors.rit.edu rpi',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
# },
'sagemath': {
#'command': '~/bin/csc-sync-standard sage mirror.clibre.uqam.ca sage',
'command': '~/bin/csc-sync-standard sage rsync.sagemath.org sage',
'interval': twice_daily,
'max-sync-time': maxtime,
},
# 'cs136': {
# 'command': '~/bin/csc-sync-ssh uw-coursewear/cs136 linux024.student.cs.uwaterloo.ca /u/cs136/mirror.uwaterloo.ca csc01 ~/.ssh/id_rsa_csc01',
# 'interval': hourly,
# 'max-sync-time': maxtime,
# },
'vlc': {
'command': '~/bin/csc-sync-standard vlc rsync.videolan.org videolan-ftp',
'interval': twice_daily,
'max-sync-time': maxtime,
},
2021-12-22 12:54:56 -05:00
# We stopped mirroring qtproject on 2021-12-22 due to disk performance issues.
#'qtproject': {
# 'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
#},
2021-09-09 00:31:28 -04:00
'tdf': {
'command': '~/bin/csc-sync-standard tdf rsync.documentfoundation.org tdf-pub',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'saltstack': {
'command': '~/bin/csc-sync-s3 saltstack https://s3.repo.saltproject.io',
'interval': daily,
'max-sync-time': maxtime,
},
# 'kali': {
# 'command': '~/bin/csc-sync-standard kali kali.mirror.globo.tech kali',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
# },
# 'kali-images': {
# 'command': '~/bin/csc-sync-standard kali-images kali.mirror.globo.tech kali-images',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
# },
'alpine': {
'command': '~/bin/csc-sync-standard alpine rsync.alpinelinux.org alpine',
'interval': hourly,
'max-sync-time': maxtime,
},
'raspbian': {
2021-10-17 15:21:15 -04:00
'command': '~/bin/csc-sync-standard raspbian raspbian.freemirror.org raspbian',
2021-09-09 00:31:28 -04:00
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'raspberrypi': {
'command': '~/bin/csc-sync-standard-ipv6 raspberrypi apt-repo.raspberrypi.org archive',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'ipfire': {
'command': '~/bin/csc-sync-standard ipfire rsync.ipfire.org full',
'interval': hourly,
'max-sync-time': maxtime,
},
'manjaro': {
'command': '~/bin/csc-sync-standard manjaro mirrorservice.org repo.manjaro.org/repos/',
'interval': hourly,
'max-sync-time': maxtime,
},
'mxlinux': {
'command': '~/bin/csc-sync-standard mxlinux mirror.math.princeton.edu pub/mxlinux/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'mxlinux-iso': {
'command': '~/bin/csc-sync-standard mxlinux-iso mirror.math.princeton.edu pub/mxlinux-iso/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'parabola': {
'command': '~/bin/csc-sync-standard parabola repo.parabola.nu:875 repos/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
#'hyperbola-sources': {
# 'command': '~/bin/csc-sync-chmod hyperbola/sources repo.hyperbola.info:52000 repo/',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
#},
#'hyperbola-stable': {
# 'command': '~/bin/csc-sync-chmod hyperbola/gnu-plus-linux-libre/stable repo.hyperbola.info:52012 repo/',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
#},
#'hyperbola-testing': {
# 'command': '~/bin/csc-sync-chmod hyperbola/gnu-plus-linux-libre/testing repo.hyperbola.info:52011 repo/',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
#},
'trisquel-packages': {
'command': '~/bin/csc-sync-standard trisquel/packages rsync.trisquel.info trisquel.packages/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'trisquel-iso': {
'command': '~/bin/csc-sync-standard trisquel/iso rsync.trisquel.info trisquel.iso/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
'almalinux': {
'command': '~/bin/csc-sync-standard almalinux rsync.repo.almalinux.org almalinux/',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'ceph': {
'command': '~/bin/csc-sync-ceph -q -s global -t ceph',
'interval': tri_hourly,
'max-sync-time': maxtime,
},
}
def mirror_status():
out = []
for x in repos:
repository = repos[x]
last_attempt = repository['last-attempt']
next_attempt = repository['last-attempt'] + repository['interval']
out.append( [x, last_attempt, next_attempt] )
out.sort(key= lambda x: x[2])
#turn floating point time values into human readable strings
for x in out:
for y in (1,2):
x[y] = time.ctime(x[y])
out.insert(0,['Repository', 'Last Synced', 'Next Expected Sync'])
#calculate maximum width of each column
widths = []
i = 0
while True:
try:
column_width = max([len(row[i]) for row in out])
widths.append(column_width + 3)
i = i + 1
except:
break
#compose table string and pad out columns
status_string = '%s%s%s\n' % (out[0][0].ljust(widths[0]), out[0][1].ljust(widths[1]), out[0][2].ljust(widths[2]))
for x in out[1:]:
for y in (0,1,2):
status_string = status_string + ('%s' % x[y].rjust(widths[y]))
status_string = status_string + '\n'
return status_string
def init_last_sync():
if not os.path.isdir('stamps'):
os.mkdir('stamps')
now = time.time()
for repo in repos:
try:
last = os.stat('stamps/%s' % repo).st_mtime
repos[repo]['last-sync'], repos[repo]['last-attempt'] = last, last
logging.info('repo %s last synced %d seconds ago' % (repo, now - last))
except OSError:
repos[repo]['last-sync'], repos[repo]['last-attempt'] = 0, 0
logging.warning('repo %s has never been synced' % repo)
def update_last_attempt(repo, start_time):
repos[repo]['last-attempt'] = start_time
def update_last_sync(repo, start_time, duration, exit_code):
repos[repo]['last-sync'] = start_time
update_last_attempt(repo, start_time)
# touch the timestamp file
with open('stamps/%s' % repo, 'w') as f:
f.write('%d,%d,%d' % (start_time, duration, exit_code))
# open('stamps/%s' % repo, 'w').close()
def handler(signum, frame):
logging.info("Caught signal %d" % signum)
os.remove(earPath)
for job in jobs:
os.kill(job, signal.SIGTERM)
time.sleep(1)
for job in jobs:
os.kill(job, signal.SIGKILL)
try:
while True:
pid, status = os.wait()
logging.info("Child process %d has terminated" % pid)
#ECHILD is thrown when there are no child processes left
except OSError, e:
if e.errno != errno.ECHILD:
raise
logging.info("All child processes terminated. Goodbye!")
sys.exit()
def setup_logging():
if not os.path.isdir('logs'):
os.mkdir('logs')
try:
os.unlink(earPath)
except OSError:
pass
if '-d' in sys.argv:
logging.basicConfig(filename="/home/mirror/merlin/logs/merlin.log",
level=logging.DEBUG, format="%(asctime)-15s %(message)s")
else:
logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s")
def sync(current, now):
pid = os.fork()
if not pid:
try:
logfd = os.open('logs/%s' % current, os.O_WRONLY|os.O_APPEND|os.O_CREAT, 0644)
nulfd = os.open('/dev/null', os.O_RDONLY)
os.dup2(nulfd, 0)
os.dup2(logfd, 1)
os.dup2(logfd, 2)
os.close(logfd)
os.close(nulfd)
os.execv("/bin/sh", ['sh', '-c', repos[current]['command']])
except OSError, e:
print >>sys.stderr, 'failed to exec: %s' % e
os.exit(1)
#There exists a race condition that manifests if merlin is asked to terminate ( by receiving signal ) after fork() but before the jobs table is updated.
#Normally you would mask off signals to avoid attempting shutdown in that time period, but this is not supported in the version of Python for which merlin was written.
#TODO:
#The Linux Programming Interface 24.5 illustrates how to synchronize parent and child using signals. Might be applicable.
jobs[pid] = {'name': current, 'start_time': now, 'status': 'running'}
def zfssync(current):
pid = os.fork()
if not pid:
try:
logfd = os.open('logs/zfssync-%s.log' % current, os.O_WRONLY|os.O_APPEND|os.O_CREAT, 0644)
nulfd = os.open('/dev/null', os.O_RDONLY)
os.dup2(nulfd, 0)
os.dup2(logfd, 1)
os.dup2(logfd, 2)
os.close(logfd)
os.close(nulfd)
os.execv("/home/mirror/bin/zfssync", ['zfssync', current])
except OSError, e:
print >>sys.stderr, 'failed to exec: %s' % e
os.exit(1)
def await_command(ear):
if select.select([ear],[],[],1) == ([ear],[],[]):
#handle command heard on ear socket
s, address = ear.accept()
cmdstring = ''
while True:
data = s.recv(cmd_buf_size)
if not data:
break
cmdstring = cmdstring + data
cmdstring = cmdstring.split(':',1)
command = cmdstring[0]
try:
if command == 'sync':
try:
arg = cmdstring[1]
if arg in repos:
if arg in (x['name'] for x in jobs.itervalues()):
logging.info('Cannot force sync: %s. Already syncing.' % arg)
s.send('Cannot force sync %s, already syncing.' % command)
else:
logging.info('Forcing sync: %s' % arg)
s.send('Forcing sync: %s' % arg)
sync(arg, time.time())
else:
logging.info('%s not tracked, cannot sync.' % arg)
s.send('%s not tracked, cannot sync.' % arg)
except:
s.send('Could not parse sync command, forced sync fails.')
raise
elif command == 'status':
s.send(mirror_status())
elif command == 'dump':
s.send(str(jobs))
s.send("\n")
s.send(str(repos))
2021-09-09 00:31:28 -04:00
else:
logging.error('Received unrecognized command: %s' % command)
s.send('Bad command: %s' % command)
s.close()
except socket.error, e:
logging.error('Could not communicate with arthur over socket.')
def new_jobs(now):
#To prevent repos at the 'bottom' of the dictionary from getting neglected when mirror is under unusual load (and merlin is running at MAX_JOBS)
keys = repos.keys()
random.shuffle(keys)
for current in keys:
2021-09-09 00:31:28 -04:00
if len(jobs) >= MAX_JOBS:
break
if now <= repos[current]['last-attempt'] + mintime:
continue
if current in (x['name'] for x in jobs.itervalues()):
continue
when_due = repos[current]['last-sync'] + repos[current]['interval']
if now >= when_due:
logging.debug("syncing %s, due for %d seconds" % (current, now - when_due))
sync(current, now)
def handle_completed(now):
try:
pid, status = os.waitpid(-1, os.WNOHANG) #waitpid() likes to throw an exception when there's nothing to wait for
#Don't know if we should be forever ignoring hung jobs if they eventually return...
if pid != 0 and pid in jobs:
job = jobs.pop(pid)
if os.WIFSIGNALED(status):
logging.error("%s %s sync terminated with signal %d" % (job['status'], job['name'], os.WTERMSIG(status)))
update_last_attempt(job['name'], job['start_time'])
elif os.WIFEXITED(status):
exit_status = os.WEXITSTATUS(status)
logging.error("%s %s sync exited with status %d " % (job['status'], job['name'], exit_status))
update_last_attempt(job['name'], job['start_time'])
interval = repos[job['name']]['interval']
sync_took = int(now - job['start_time'])
next_sync = max(0, interval - sync_took)
update_last_sync(job['name'], job['start_time'], sync_took, exit_status)
logging.info('%s sync complete, took %d seconds, syncs again in %d seconds'
% (job['name'], sync_took, next_sync))
zfssync(job['name'])
except OSError, e:
if e.errno != errno.ECHILD:
raise
def check_hung(now):
for pid in jobs:
#check if hung
runtime = now - jobs[pid]['start_time']
repo = jobs[pid]['name']
if jobs[pid]['status'] == 'running' and runtime > repos[repo]['max-sync-time']:
jobs[pid]['status'] = 'hung'
logging.error("%s sync process hung, process pid %d" % (repo, pid))
def main():
if not os.geteuid():
print "Don't run merlin as root!"
sys.exit(1);
setup_logging()
logging.info("Starting Merlin")
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGINT, handler)
init_last_sync()
old_umask = os.umask(0o002)
ear = socket.socket(socket.AF_UNIX)
ear.bind(earPath)
ear.listen(1)
os.umask(old_umask)
os.chown(earPath, -1, grp.getgrnam("push").gr_gid)
while True:
await_command(ear)
now = time.time()
new_jobs(now)
handle_completed(now)
check_hung(now)
if __name__ == '__main__':
main()