mirror/merlin_old/merlin.py

#!/usr/bin/python2

import time, sys, os, errno, logging, signal, copy, select, socket, grp, random

daily = 86400
twice_daily = 86400 / 2
hourly = 3600
bi_hourly = 7200
tri_hourly = 10800
twice_hourly = 1800
ten_minutely = 600
five_minutely = 300
maxtime = 86400
mintime = 60
jobs = {}
MAX_JOBS = 6
#earPath = '/home/mirror/merlin/merlin.sock'
earPath = '/mirror/merlin/run/merlin.sock'
cmd_buf_size = 4096

repos = {
    'debian': {
        'command': '~/bin/csc-sync-debian debian debian.mirror.rafal.ca debian',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
#    'debian-cdimage': {
#        'command': '~/bin/csc-sync-cdimage debian-cdimage cdimage.debian.org cdimage',
#        'interval': twice_daily,
#        'max-sync-time': maxtime,
#    },
    'ubuntu': {
        'command': '~/bin/csc-sync-debian ubuntu archive.ubuntu.com ubuntu drescher.canonical.com',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'ubuntu-ports': {
        'command': '~/bin/csc-sync-debian ubuntu-ports ports.ubuntu.com ubuntu-ports drescher.canonical.com',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'linuxmint-packages': {
        'command': '~/bin/csc-sync-debian linuxmint-packages rsync-packages.linuxmint.com packages',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'debian-multimedia': {
        'command': '~/bin/csc-sync-debian debian-multimedia www.deb-multimedia.org deb',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'debian-backports': {
        'command': '~/bin/csc-sync-debian debian-backports debian.mirror.rafal.ca debian-backports',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
#    'debian-volatile': {
#        'command': '~/bin/csc-sync-debian debian-volatile debian.mirror.rafal.ca debian-volatile',
#        'interval': bi_hourly,
#        'max-sync-time': maxtime,
#    },
    'debian-security': {
        'command': '~/bin/csc-sync-debian debian-security rsync.security.debian.org debian-security security-master.debian.org',
        'interval': twice_hourly,
        'max-sync-time': maxtime,
    },
    'ubuntu-releases': {
        'command': '~/bin/csc-sync-standard ubuntu-releases rsync.releases.ubuntu.com releases',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'xubuntu-releases': {
        'command': '~/bin/csc-sync-standard xubuntu-releases cdimage.ubuntu.com cdimage/xubuntu/releases/',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
#    'emdebian': {
#        'command': '~/bin/csc-sync-badperms emdebian www.emdebian.org debian',
#        'interval': twice_daily,
#        'max-sync-time': maxtime,
#    },
    'puppylinux': {
        'command': '~/bin/csc-sync-standard puppylinux distro.ibiblio.org puppylinux',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'CPAN': {
        'command': '~/bin/csc-sync-standard CPAN cpan-rsync.perl.org CPAN',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'CRAN': {
        'command': '~/bin/csc-sync-ssh CRAN cran.r-project.org "" cran-rsync ~/.ssh/id_cran_rsa',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'CTAN': {
        'command': '~/bin/csc-sync-standard CTAN rsync.dante.ctan.org CTAN',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
#    'openoffice': {
#        'command': '~/bin/csc-sync-standard openoffice rsync.services.openoffice.org openoffice-extended',
#        'command': '~/bin/csc-sync-standard openoffice ftp.snt.utwente.nl openoffice-extended',
#        'interval': twice_daily,
#        'max-sync-time': maxtime,
#    },
    'fedora-epel': {
        'command': '~/bin/csc-sync-standard fedora/epel mirrors.kernel.org fedora-epel && ~/bin/report_mirror >/dev/null',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'cygwin': {
        'command': '~/bin/csc-sync-standard cygwin cygwin.com cygwin-ftp',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'gnu': {
        #'command': '~/bin/csc-sync-standard gnu mirrors.ibiblio.org gnuftp/gnu/',
        'command': '~/bin/csc-sync-standard gnu ftp.gnu.org gnu',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'nongnu': {
        # 'command': '~/bin/csc-sync-standard nongnu dl.sv.gnu.org releases --ignore-errors',
        'command': '~/bin/csc-sync-standard nongnu dl.sv.gnu.org releases',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'mysql': {
        #'command': '~/bin/csc-sync-standard mysql mysql.he.net mysql',
        'command': '~/bin/csc-sync-standard mysql rsync.mirrorservice.org ftp.mysql.com',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
#	No longer syncs, and no longer really relevant
#    'mozdev': {
#        'command': '~/bin/csc-sync-standard mozdev rsync.mozdev.org mozdev',
#        'interval': twice_daily,
#        'max-sync-time': maxtime,
#    },
    'gnome': {
        'command': '~/bin/csc-sync-standard gnome master.gnome.org gnomeftp gnome',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'damnsmalllinux': {
        'command': '~/bin/csc-sync-standard damnsmalllinux ftp.heanet.ie mirrors/damnsmalllinux.org/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'linuxmint': {
        'command': '~/bin/csc-sync-standard linuxmint pub.linuxmint.com pub',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'kernel.org-linux': {
        'command': '~/bin/csc-sync-standard kernel.org/linux rsync.kernel.org pub/linux/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'kernel.org-software': {
        'command': '~/bin/csc-sync-standard kernel.org/software rsync.kernel.org pub/software/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'apache': {
        'command': '~/bin/csc-sync-apache apache rsync.us.apache.org apache-dist',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'eclipse': {
        'command': '~/bin/csc-sync-standard eclipse download.eclipse.org eclipseMirror',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'kde': {
        'command': '~/bin/csc-sync-standard kde rsync.kde.org kdeftp',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'kde-applicationdata': {
        'command': '~/bin/csc-sync-standard kde-applicationdata rsync.kde.org applicationdata',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    # We are a Tier 1 arch mirror (https://bugs.archlinux.org/task/52853)
    # so our IP is important.
    'archlinux': {
        #'command': '~/bin/csc-sync-standard archlinux archlinux.mirror.rafal.ca archlinux',
        'command': '~/bin/csc-sync-archlinux archlinux',
        'interval': five_minutely,
        'max-sync-time': maxtime,
    },
    'artixlinux': {
        'command': '~/bin/csc-sync-standard artixlinux mirror1.artixlinux.org artix-linux',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    # We stopped mirroring debian-ports on 2021-12-22 due to disk performance issues.
    #'debian-ports': {
    #    'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports',
    #    'interval': twice_daily,
    #    'max-sync-time': maxtime,
    #},
    'slackware': {
        'command': '~/bin/csc-sync-standard slackware slackware.cs.utah.edu slackware',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'debian-cd': {
        'command': '~/bin/csc-sync-debian-cd',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'x.org': {
        #'command': '~/bin/csc-sync-standard x.org xorg.freedesktop.org xorg-archive',
        #'command': '~/bin/csc-sync-standard x.org mirror.us.leaseweb.net xorg',
        'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'centos': {
        'command': '~/bin/csc-sync-standard centos us-msync.centos.org CentOS',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'opensuse': {
        'command': '~/bin/csc-sync-standard opensuse stage.opensuse.org opensuse-full/opensuse/ #"--exclude distribution/.timestamp_invisible"',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'FreeBSD': {
        # Has not updated since at least June 2018
        #'command': '~/bin/csc-sync-standard FreeBSD ftp10.us.freebsd.org FreeBSD',
        'command': '~/bin/csc-sync-standard FreeBSD ftp2.uk.freebsd.org ftp.freebsd.org/pub/FreeBSD/',
        #'command': '~/bin/csc-sync-standard FreeBSD ftp3.us.freebsd.org FreeBSD/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'fedora-enchilada': {
        # 'command': '~/bin/csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null',
        'command': '~/bin/csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ && ~/bin/report_mirror >/dev/null',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'ubuntu-ports-releases': {
        'command': '~/bin/csc-sync-standard ubuntu-ports-releases cdimage.ubuntu.com cdimage/releases/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'gentoo-distfiles': {
        'command': '~/bin/csc-sync-gentoo',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'gentoo-portage': {
        'command': '~/bin/csc-sync-standard gentoo-portage rsync1.us.gentoo.org gentoo-portage',
        'interval': twice_hourly,
        'max-sync-time': maxtime,
    },
    # This project is no longer available for mirroring
    # https://bugzilla.mozilla.org/show_bug.cgi?id=807543
    #'mozilla.org': {
    #    'command': '~/bin/csc-sync-standard mozilla.org releases-rsync.mozilla.org mozilla-releases',
    #    'interval': twice_hourly,
    #    'max-sync-time': maxtime,
    #},
    'gutenberg': {
        'command': '~/bin/csc-sync-standard gutenberg ftp@ftp.ibiblio.org gutenberg',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'racket-installers': {
        'command': '~/bin/csc-sync-wget racket/racket-installers https://mirror.racket-lang.org/installers/ 1',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'plt-bundles': {
        'command': '~/bin/csc-sync-standard racket/plt-bundles mirror.racket-lang.org plt-bundles',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    # We stopped mirroring OpenBSD on 2021-12-22 due to degraded disk performance
    #'OpenBSD': {
    #    'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp',
    #    'interval': twice_daily,
    #    'max-sync-time': maxtime,
    #},
    'xiph': {
        #'command': '~/bin/csc-sync-standard xiph downloads.xiph.org xiph/releases',
        'command': '~/bin/csc-sync-standard xiph ftp.osuosl.org xiph',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
#	We currently don't have the disk space
    'netbsd': {
        'command': '~/bin/csc-sync-standard NetBSD rsync.netbsd.org NetBSD',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    # We stopped mirroring pkgsrc on 2021-12-22 due to disk performance issues.
    #'netbsd-pkgsrc': {
    #    'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc',
    #    #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/',
    #    'interval': twice_daily,
    #    'max-sync-time': maxtime,
    #},
    'macports-release': {
        'command': '~/bin/csc-sync-standard MacPorts/release rsync.macports.org macports/release/',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'macports-distfiles': {
        'command': '~/bin/csc-sync-standard MacPorts/mpdistfiles rsync.macports.org macports/distfiles/',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
#    'raspberrypi': {
#        'command': '~/bin/csc-sync-standard raspberrypi mirrors.rit.edu rpi',
#        'interval': twice_daily,
#        'max-sync-time': maxtime,
#    },
    'sagemath': {
        #'command': '~/bin/csc-sync-standard sage mirror.clibre.uqam.ca sage',
        'command': '~/bin/csc-sync-standard sage rsync.sagemath.org sage',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
#    'cs136': {
#        'command': '~/bin/csc-sync-ssh uw-coursewear/cs136 linux024.student.cs.uwaterloo.ca /u/cs136/mirror.uwaterloo.ca csc01 ~/.ssh/id_rsa_csc01',
#        'interval': hourly,
#        'max-sync-time': maxtime,
#    },
    'vlc': {
        'command': '~/bin/csc-sync-standard vlc rsync.videolan.org videolan-ftp',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    # We stopped mirroring qtproject on 2021-12-22 due to disk performance issues.
    #'qtproject': {
    #    'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all',
    #    'interval': twice_daily,
    #    'max-sync-time': maxtime,
    #},
    'tdf': {
        'command': '~/bin/csc-sync-standard tdf rsync.documentfoundation.org tdf-pub',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'saltstack': {
        'command': '~/bin/csc-sync-s3 saltstack https://s3.repo.saltproject.io',
        'interval': daily,
        'max-sync-time': maxtime,
    },
#    'kali': {
#        'command': '~/bin/csc-sync-standard kali kali.mirror.globo.tech kali',
#        'interval': twice_daily,
#        'max-sync-time': maxtime,
#    },
#    'kali-images': {
#        'command': '~/bin/csc-sync-standard kali-images kali.mirror.globo.tech kali-images',
#        'interval': twice_daily,
#        'max-sync-time': maxtime,
#    },
    'alpine': {
        'command': '~/bin/csc-sync-standard alpine rsync.alpinelinux.org alpine',
        'interval': hourly,
        'max-sync-time': maxtime,
    },
    'raspbian': {
        'command': '~/bin/csc-sync-standard raspbian raspbian.freemirror.org raspbian',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'raspberrypi': {
        'command': '~/bin/csc-sync-standard-ipv6 raspberrypi apt-repo.raspberrypi.org archive',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'ipfire': {
        'command': '~/bin/csc-sync-standard ipfire rsync.ipfire.org full',
        'interval': hourly,
        'max-sync-time': maxtime,
    },
    'manjaro': {
        'command': '~/bin/csc-sync-standard manjaro mirrorservice.org repo.manjaro.org/repos/',
        'interval': hourly,
        'max-sync-time': maxtime,
    },
    'mxlinux': {
        'command': '~/bin/csc-sync-standard mxlinux mirror.math.princeton.edu pub/mxlinux/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'mxlinux-iso': {
        'command': '~/bin/csc-sync-standard mxlinux-iso mirror.math.princeton.edu pub/mxlinux-iso/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },

    'parabola': {
        'command': '~/bin/csc-sync-standard parabola repo.parabola.nu:875 repos/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    #'hyperbola-sources': {
    #    'command': '~/bin/csc-sync-chmod hyperbola/sources repo.hyperbola.info:52000 repo/',
    #    'interval': twice_daily,
    #    'max-sync-time': maxtime,
    #},
    #'hyperbola-stable': {
    #    'command': '~/bin/csc-sync-chmod hyperbola/gnu-plus-linux-libre/stable repo.hyperbola.info:52012 repo/',
    #    'interval': twice_daily,
    #    'max-sync-time': maxtime,
    #},
    #'hyperbola-testing': {
    #    'command': '~/bin/csc-sync-chmod hyperbola/gnu-plus-linux-libre/testing repo.hyperbola.info:52011 repo/',
    #    'interval': twice_daily,
    #    'max-sync-time': maxtime,
    #},
    'trisquel-packages': {
        'command': '~/bin/csc-sync-standard trisquel/packages rsync.trisquel.info trisquel.packages/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'trisquel-iso': {
        'command': '~/bin/csc-sync-standard trisquel/iso rsync.trisquel.info trisquel.iso/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
    'almalinux': {
        'command': '~/bin/csc-sync-standard almalinux rsync.repo.almalinux.org almalinux/',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
    'ceph': {
        'command': '~/bin/csc-sync-ceph -q -s global -t ceph',
        'interval': tri_hourly,
        'max-sync-time': maxtime,
    },
}


def mirror_status():
    out = []
    for x in repos:
        repository = repos[x]
        last_attempt = repository['last-attempt']
        next_attempt = repository['last-attempt'] + repository['interval']
        out.append( [x, last_attempt, next_attempt] )

    out.sort(key= lambda x: x[2])

    #turn floating point time values into human readable strings
    for x in out:
        for y in (1,2):
            x[y] = time.ctime(x[y])

    out.insert(0,['Repository', 'Last Synced', 'Next Expected Sync'])

    #calculate maximum width of each column
    widths = []
    i = 0
    while True:
        try:
            column_width = max([len(row[i]) for row in out])
            widths.append(column_width + 3)
            i = i + 1
        except:
            break

    #compose table string and pad out columns
    status_string = '%s%s%s\n' % (out[0][0].ljust(widths[0]), out[0][1].ljust(widths[1]), out[0][2].ljust(widths[2]))
    for x in out[1:]:
        for y in (0,1,2):
            status_string = status_string + ('%s' % x[y].rjust(widths[y]))
        status_string = status_string + '\n'

    return status_string


def init_last_sync():

    if not os.path.isdir('stamps'):
        os.mkdir('stamps')

    now = time.time()

    for repo in repos:
        try:
            last = os.stat('stamps/%s' % repo).st_mtime
            repos[repo]['last-sync'], repos[repo]['last-attempt']  = last, last
            logging.info('repo %s last synced %d seconds ago' % (repo, now - last))
        except OSError:
            repos[repo]['last-sync'], repos[repo]['last-attempt']  = 0, 0
            logging.warning('repo %s has never been synced' % repo)


def update_last_attempt(repo, start_time):
    repos[repo]['last-attempt'] = start_time

def update_last_sync(repo, start_time, duration, exit_code):

    repos[repo]['last-sync'] = start_time
    update_last_attempt(repo, start_time)

    # touch the timestamp file
    with open('stamps/%s' % repo, 'w') as f:
        f.write('%d,%d,%d' % (start_time, duration, exit_code))
    # open('stamps/%s' % repo, 'w').close()

def handler(signum, frame):
    logging.info("Caught signal %d" % signum)

    os.remove(earPath)

    for job in jobs:
        os.kill(job, signal.SIGTERM)

    time.sleep(1)

    for job in jobs:
        os.kill(job, signal.SIGKILL)

    try:
        while True:
            pid, status = os.wait()
            logging.info("Child process %d has terminated" % pid)

    #ECHILD is thrown when there are no child processes left
    except OSError, e:
        if e.errno != errno.ECHILD:
            raise
        logging.info("All child processes terminated. Goodbye!")
        sys.exit()

def setup_logging():
    if not os.path.isdir('logs'):
        os.mkdir('logs')

    try:
        os.unlink(earPath)
    except OSError:
        pass

    if '-d' in sys.argv:
        logging.basicConfig(filename="/home/mirror/merlin/logs/merlin.log",
                            level=logging.DEBUG, format="%(asctime)-15s %(message)s")
    else:
        logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s")

def sync(current, now):
    pid = os.fork()
    if not pid:
        try:
            logfd = os.open('logs/%s' % current, os.O_WRONLY|os.O_APPEND|os.O_CREAT, 0644)
            nulfd = os.open('/dev/null', os.O_RDONLY)
            os.dup2(nulfd, 0)
            os.dup2(logfd, 1)
            os.dup2(logfd, 2)
            os.close(logfd)
            os.close(nulfd)
            os.execv("/bin/sh", ['sh', '-c', repos[current]['command']])
        except OSError, e:
            print >>sys.stderr, 'failed to exec: %s' % e
            os.exit(1)

    #There exists a race condition that manifests if merlin is asked to terminate ( by receiving signal ) after fork() but before the jobs table is updated.
    #Normally you would mask off signals to avoid attempting shutdown in that time period, but this is not supported in the version of Python for which merlin was written.
    #TODO:
    #The Linux Programming Interface 24.5 illustrates how to synchronize parent and child using signals.  Might be applicable.

    jobs[pid] = {'name': current, 'start_time': now, 'status': 'running'}

def zfssync(current):
    pid = os.fork()
    if not pid:
        try:
            logfd = os.open('logs/zfssync-%s.log' % current, os.O_WRONLY|os.O_APPEND|os.O_CREAT, 0644)
            nulfd = os.open('/dev/null', os.O_RDONLY)
            os.dup2(nulfd, 0)
            os.dup2(logfd, 1)
            os.dup2(logfd, 2)
            os.close(logfd)
            os.close(nulfd)
            os.execv("/home/mirror/bin/zfssync", ['zfssync', current])
        except OSError, e:
            print >>sys.stderr, 'failed to exec: %s' % e
            os.exit(1)

def await_command(ear):
    if select.select([ear],[],[],1) == ([ear],[],[]):
        #handle command heard on ear socket
        s, address = ear.accept()
        cmdstring = ''
        while True:
            data = s.recv(cmd_buf_size)
            if not data:
                break
            cmdstring = cmdstring + data
        cmdstring = cmdstring.split(':',1)
        command = cmdstring[0]
        try:
            if command == 'sync':
                try:
                    arg = cmdstring[1]
                    if arg in repos:
                        if arg in (x['name'] for x in jobs.itervalues()):
                            logging.info('Cannot force sync: %s. Already syncing.' % arg)
                            s.send('Cannot force sync %s, already syncing.' % command)
                        else:
                            logging.info('Forcing sync: %s' % arg)
                            s.send('Forcing sync: %s' % arg)
                            sync(arg, time.time())
                    else:
                        logging.info('%s not tracked, cannot sync.' % arg)
                        s.send('%s not tracked, cannot sync.' % arg)
                except:
                    s.send('Could not parse sync command, forced sync fails.')
                    raise

            elif command == 'status':
                s.send(mirror_status())

            elif command == 'dump':
                s.send(str(jobs))
                s.send("\n")
                s.send(str(repos))

            else:
                logging.error('Received unrecognized command: %s' % command)
                s.send('Bad command: %s' % command)

            s.close()

        except socket.error, e:
            logging.error('Could not communicate with arthur over socket.')

def new_jobs(now):
    #To prevent repos at the 'bottom' of the dictionary from getting neglected when mirror is under unusual load (and merlin is running at MAX_JOBS)
    keys = repos.keys()
    random.shuffle(keys)
    for current in keys:
        if len(jobs) >= MAX_JOBS:
            break
        if now <= repos[current]['last-attempt'] + mintime:
            continue
        if current in (x['name'] for x in jobs.itervalues()):
            continue
        when_due = repos[current]['last-sync'] + repos[current]['interval']
        if now >= when_due:
            logging.debug("syncing %s, due for %d seconds" % (current, now - when_due))
            sync(current, now)

def handle_completed(now):
    try:
        pid, status = os.waitpid(-1, os.WNOHANG) #waitpid() likes to throw an exception when there's nothing to wait for

        #Don't know if we should be forever ignoring hung jobs if they eventually return...
        if pid != 0 and pid in jobs:
            job = jobs.pop(pid)

            if os.WIFSIGNALED(status):
                logging.error("%s %s sync terminated with signal %d" % (job['status'], job['name'], os.WTERMSIG(status)))
                update_last_attempt(job['name'], job['start_time'])
            elif os.WIFEXITED(status):
                exit_status = os.WEXITSTATUS(status)
                logging.error("%s %s sync exited with status %d " % (job['status'], job['name'], exit_status))
                update_last_attempt(job['name'], job['start_time'])
                interval = repos[job['name']]['interval']
                sync_took = int(now - job['start_time'])
                next_sync = max(0, interval - sync_took)
                update_last_sync(job['name'], job['start_time'], sync_took, exit_status)
                logging.info('%s sync complete, took %d seconds, syncs again in %d seconds'
                             % (job['name'], sync_took, next_sync))

                zfssync(job['name'])

    except OSError, e:
        if e.errno != errno.ECHILD:
            raise

def check_hung(now):
    for pid in jobs:
        #check if hung
        runtime = now - jobs[pid]['start_time']
        repo = jobs[pid]['name']
        if jobs[pid]['status'] == 'running' and runtime > repos[repo]['max-sync-time']:
            jobs[pid]['status'] = 'hung'
            logging.error("%s sync process hung, process pid %d" % (repo, pid))

def main():
    if not os.geteuid():
        print "Don't run merlin as root!"
        sys.exit(1);

    setup_logging()

    logging.info("Starting Merlin")

    signal.signal(signal.SIGTERM, handler)
    signal.signal(signal.SIGINT, handler)

    init_last_sync()


    old_umask = os.umask(0o002)
    ear = socket.socket(socket.AF_UNIX)
    ear.bind(earPath)
    ear.listen(1)
    os.umask(old_umask)
    os.chown(earPath, -1, grp.getgrnam("push").gr_gid)

    while True:
        await_command(ear)

        now = time.time()

        new_jobs(now)

        handle_completed(now)

        check_hung(now)


if __name__ == '__main__':
    main()