729 lines
26 KiB
Python
Executable File
729 lines
26 KiB
Python
Executable File
#!/usr/bin/python2
|
|
|
|
import time, sys, os, errno, logging, signal, copy, select, socket, grp, random
|
|
|
|
daily = 86400
|
|
twice_daily = 86400 / 2
|
|
hourly = 3600
|
|
bi_hourly = 7200
|
|
tri_hourly = 10800
|
|
twice_hourly = 1800
|
|
ten_minutely = 600
|
|
five_minutely = 300
|
|
maxtime = 86400
|
|
mintime = 60
|
|
jobs = {}
|
|
MAX_JOBS = 6
|
|
#earPath = '/home/mirror/merlin/merlin.sock'
|
|
earPath = '/mirror/merlin/run/merlin.sock'
|
|
cmd_buf_size = 4096
|
|
|
|
repos = {
|
|
'debian': {
|
|
'command': '~/bin/csc-sync-debian debian debian.mirror.rafal.ca debian',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# 'debian-cdimage': {
|
|
# 'command': '~/bin/csc-sync-cdimage debian-cdimage cdimage.debian.org cdimage',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
# },
|
|
'ubuntu': {
|
|
'command': '~/bin/csc-sync-debian ubuntu archive.ubuntu.com ubuntu drescher.canonical.com',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'ubuntu-ports': {
|
|
'command': '~/bin/csc-sync-debian ubuntu-ports ports.ubuntu.com ubuntu-ports drescher.canonical.com',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'linuxmint-packages': {
|
|
'command': '~/bin/csc-sync-debian linuxmint-packages rsync-packages.linuxmint.com packages',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'debian-multimedia': {
|
|
'command': '~/bin/csc-sync-debian debian-multimedia www.deb-multimedia.org deb',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'debian-backports': {
|
|
'command': '~/bin/csc-sync-debian debian-backports debian.mirror.rafal.ca debian-backports',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# 'debian-volatile': {
|
|
# 'command': '~/bin/csc-sync-debian debian-volatile debian.mirror.rafal.ca debian-volatile',
|
|
# 'interval': bi_hourly,
|
|
# 'max-sync-time': maxtime,
|
|
# },
|
|
'debian-security': {
|
|
'command': '~/bin/csc-sync-debian debian-security rsync.security.debian.org debian-security security-master.debian.org',
|
|
'interval': twice_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'ubuntu-releases': {
|
|
'command': '~/bin/csc-sync-standard ubuntu-releases rsync.releases.ubuntu.com releases',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'xubuntu-releases': {
|
|
'command': '~/bin/csc-sync-standard xubuntu-releases cdimage.ubuntu.com cdimage/xubuntu/releases/',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# 'emdebian': {
|
|
# 'command': '~/bin/csc-sync-badperms emdebian www.emdebian.org debian',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
# },
|
|
'puppylinux': {
|
|
'command': '~/bin/csc-sync-standard puppylinux distro.ibiblio.org puppylinux',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'CPAN': {
|
|
'command': '~/bin/csc-sync-standard CPAN cpan-rsync.perl.org CPAN',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'CRAN': {
|
|
'command': '~/bin/csc-sync-ssh CRAN cran.r-project.org "" cran-rsync ~/.ssh/id_cran_rsa',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'CTAN': {
|
|
'command': '~/bin/csc-sync-standard CTAN rsync.dante.ctan.org CTAN',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# 'openoffice': {
|
|
# 'command': '~/bin/csc-sync-standard openoffice rsync.services.openoffice.org openoffice-extended',
|
|
# 'command': '~/bin/csc-sync-standard openoffice ftp.snt.utwente.nl openoffice-extended',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
# },
|
|
'fedora-epel': {
|
|
'command': '~/bin/csc-sync-standard fedora/epel mirrors.kernel.org fedora-epel && ~/bin/report_mirror >/dev/null',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'cygwin': {
|
|
'command': '~/bin/csc-sync-standard cygwin cygwin.com cygwin-ftp',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'gnu': {
|
|
#'command': '~/bin/csc-sync-standard gnu mirrors.ibiblio.org gnuftp/gnu/',
|
|
'command': '~/bin/csc-sync-standard gnu ftp.gnu.org gnu',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'nongnu': {
|
|
# 'command': '~/bin/csc-sync-standard nongnu dl.sv.gnu.org releases --ignore-errors',
|
|
'command': '~/bin/csc-sync-standard nongnu dl.sv.gnu.org releases',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'mysql': {
|
|
#'command': '~/bin/csc-sync-standard mysql mysql.he.net mysql',
|
|
'command': '~/bin/csc-sync-standard mysql rsync.mirrorservice.org ftp.mysql.com',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# No longer syncs, and no longer really relevant
|
|
# 'mozdev': {
|
|
# 'command': '~/bin/csc-sync-standard mozdev rsync.mozdev.org mozdev',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
# },
|
|
'gnome': {
|
|
'command': '~/bin/csc-sync-standard gnome master.gnome.org gnomeftp gnome',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'damnsmalllinux': {
|
|
'command': '~/bin/csc-sync-standard damnsmalllinux ftp.heanet.ie mirrors/damnsmalllinux.org/',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'linuxmint': {
|
|
'command': '~/bin/csc-sync-standard linuxmint pub.linuxmint.com pub',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'kernel.org-linux': {
|
|
'command': '~/bin/csc-sync-standard kernel.org/linux rsync.kernel.org pub/linux/',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'kernel.org-software': {
|
|
'command': '~/bin/csc-sync-standard kernel.org/software rsync.kernel.org pub/software/',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'apache': {
|
|
'command': '~/bin/csc-sync-apache apache rsync.us.apache.org apache-dist',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'eclipse': {
|
|
'command': '~/bin/csc-sync-standard eclipse download.eclipse.org eclipseMirror',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'kde': {
|
|
'command': '~/bin/csc-sync-standard kde rsync.kde.org kdeftp',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'kde-applicationdata': {
|
|
'command': '~/bin/csc-sync-standard kde-applicationdata rsync.kde.org applicationdata',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# We are a Tier 1 arch mirror (https://bugs.archlinux.org/task/52853)
|
|
# so our IP is important.
|
|
'archlinux': {
|
|
#'command': '~/bin/csc-sync-standard archlinux archlinux.mirror.rafal.ca archlinux',
|
|
'command': '~/bin/csc-sync-archlinux archlinux',
|
|
'interval': five_minutely,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'artixlinux': {
|
|
'command': '~/bin/csc-sync-standard artixlinux mirror1.artixlinux.org artix-linux',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# We stopped mirroring debian-ports on 2021-12-22 due to disk performance issues.
|
|
#'debian-ports': {
|
|
# 'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
#},
|
|
'slackware': {
|
|
'command': '~/bin/csc-sync-standard slackware slackware.cs.utah.edu slackware',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'debian-cd': {
|
|
'command': '~/bin/csc-sync-debian-cd',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'x.org': {
|
|
#'command': '~/bin/csc-sync-standard x.org xorg.freedesktop.org xorg-archive',
|
|
#'command': '~/bin/csc-sync-standard x.org mirror.us.leaseweb.net xorg',
|
|
'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub/',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'centos': {
|
|
'command': '~/bin/csc-sync-standard centos us-msync.centos.org CentOS',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'opensuse': {
|
|
'command': '~/bin/csc-sync-standard opensuse stage.opensuse.org opensuse-full/opensuse/ #"--exclude distribution/.timestamp_invisible"',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'FreeBSD': {
|
|
# Has not updated since at least June 2018
|
|
#'command': '~/bin/csc-sync-standard FreeBSD ftp10.us.freebsd.org FreeBSD',
|
|
'command': '~/bin/csc-sync-standard FreeBSD ftp2.uk.freebsd.org ftp.freebsd.org/pub/FreeBSD/',
|
|
#'command': '~/bin/csc-sync-standard FreeBSD ftp3.us.freebsd.org FreeBSD/',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'fedora-enchilada': {
|
|
# 'command': '~/bin/csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null',
|
|
'command': '~/bin/csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ && ~/bin/report_mirror >/dev/null',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'ubuntu-ports-releases': {
|
|
'command': '~/bin/csc-sync-standard ubuntu-ports-releases cdimage.ubuntu.com cdimage/releases/',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'gentoo-distfiles': {
|
|
'command': '~/bin/csc-sync-gentoo',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'gentoo-portage': {
|
|
'command': '~/bin/csc-sync-standard gentoo-portage rsync1.us.gentoo.org gentoo-portage',
|
|
'interval': twice_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# This project is no longer available for mirroring
|
|
# https://bugzilla.mozilla.org/show_bug.cgi?id=807543
|
|
#'mozilla.org': {
|
|
# 'command': '~/bin/csc-sync-standard mozilla.org releases-rsync.mozilla.org mozilla-releases',
|
|
# 'interval': twice_hourly,
|
|
# 'max-sync-time': maxtime,
|
|
#},
|
|
'gutenberg': {
|
|
'command': '~/bin/csc-sync-standard gutenberg ftp@ftp.ibiblio.org gutenberg',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'racket-installers': {
|
|
'command': '~/bin/csc-sync-wget racket/racket-installers https://mirror.racket-lang.org/installers/ 1',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'plt-bundles': {
|
|
'command': '~/bin/csc-sync-standard racket/plt-bundles mirror.racket-lang.org plt-bundles',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# We stopped mirroring OpenBSD on 2021-12-22 due to degraded disk performance
|
|
#'OpenBSD': {
|
|
# 'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
#},
|
|
'xiph': {
|
|
#'command': '~/bin/csc-sync-standard xiph downloads.xiph.org xiph/releases',
|
|
'command': '~/bin/csc-sync-standard xiph ftp.osuosl.org xiph',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# We currently don't have the disk space
|
|
'netbsd': {
|
|
'command': '~/bin/csc-sync-standard NetBSD rsync.netbsd.org NetBSD',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# We stopped mirroring pkgsrc on 2021-12-22 due to disk performance issues.
|
|
#'netbsd-pkgsrc': {
|
|
# 'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc',
|
|
# #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
#},
|
|
'macports-release': {
|
|
'command': '~/bin/csc-sync-standard MacPorts/release rsync.macports.org macports/release/',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'macports-distfiles': {
|
|
'command': '~/bin/csc-sync-standard MacPorts/mpdistfiles rsync.macports.org macports/distfiles/',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# 'raspberrypi': {
|
|
# 'command': '~/bin/csc-sync-standard raspberrypi mirrors.rit.edu rpi',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
# },
|
|
'sagemath': {
|
|
#'command': '~/bin/csc-sync-standard sage mirror.clibre.uqam.ca sage',
|
|
'command': '~/bin/csc-sync-standard sage rsync.sagemath.org sage',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# 'cs136': {
|
|
# 'command': '~/bin/csc-sync-ssh uw-coursewear/cs136 linux024.student.cs.uwaterloo.ca /u/cs136/mirror.uwaterloo.ca csc01 ~/.ssh/id_rsa_csc01',
|
|
# 'interval': hourly,
|
|
# 'max-sync-time': maxtime,
|
|
# },
|
|
'vlc': {
|
|
'command': '~/bin/csc-sync-standard vlc rsync.videolan.org videolan-ftp',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# We stopped mirroring qtproject on 2021-12-22 due to disk performance issues.
|
|
#'qtproject': {
|
|
# 'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
#},
|
|
'tdf': {
|
|
'command': '~/bin/csc-sync-standard tdf rsync.documentfoundation.org tdf-pub',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'saltstack': {
|
|
'command': '~/bin/csc-sync-s3 saltstack https://s3.repo.saltproject.io',
|
|
'interval': daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
# 'kali': {
|
|
# 'command': '~/bin/csc-sync-standard kali kali.mirror.globo.tech kali',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
# },
|
|
# 'kali-images': {
|
|
# 'command': '~/bin/csc-sync-standard kali-images kali.mirror.globo.tech kali-images',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
# },
|
|
'alpine': {
|
|
'command': '~/bin/csc-sync-standard alpine rsync.alpinelinux.org alpine',
|
|
'interval': hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'raspbian': {
|
|
'command': '~/bin/csc-sync-standard raspbian raspbian.freemirror.org raspbian',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'raspberrypi': {
|
|
'command': '~/bin/csc-sync-standard-ipv6 raspberrypi apt-repo.raspberrypi.org archive',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'ipfire': {
|
|
'command': '~/bin/csc-sync-standard ipfire rsync.ipfire.org full',
|
|
'interval': hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'manjaro': {
|
|
'command': '~/bin/csc-sync-standard manjaro mirrorservice.org repo.manjaro.org/repos/',
|
|
'interval': hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'mxlinux': {
|
|
'command': '~/bin/csc-sync-standard mxlinux mirror.math.princeton.edu pub/mxlinux/',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'mxlinux-iso': {
|
|
'command': '~/bin/csc-sync-standard mxlinux-iso mirror.math.princeton.edu pub/mxlinux-iso/',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
|
|
'parabola': {
|
|
'command': '~/bin/csc-sync-standard parabola repo.parabola.nu:875 repos/',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
#'hyperbola-sources': {
|
|
# 'command': '~/bin/csc-sync-chmod hyperbola/sources repo.hyperbola.info:52000 repo/',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
#},
|
|
#'hyperbola-stable': {
|
|
# 'command': '~/bin/csc-sync-chmod hyperbola/gnu-plus-linux-libre/stable repo.hyperbola.info:52012 repo/',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
#},
|
|
#'hyperbola-testing': {
|
|
# 'command': '~/bin/csc-sync-chmod hyperbola/gnu-plus-linux-libre/testing repo.hyperbola.info:52011 repo/',
|
|
# 'interval': twice_daily,
|
|
# 'max-sync-time': maxtime,
|
|
#},
|
|
'trisquel-packages': {
|
|
'command': '~/bin/csc-sync-standard trisquel/packages rsync.trisquel.info trisquel.packages/',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'trisquel-iso': {
|
|
'command': '~/bin/csc-sync-standard trisquel/iso rsync.trisquel.info trisquel.iso/',
|
|
'interval': twice_daily,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'almalinux': {
|
|
'command': '~/bin/csc-sync-standard almalinux rsync.repo.almalinux.org almalinux/',
|
|
'interval': bi_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
'ceph': {
|
|
'command': '~/bin/csc-sync-ceph -q -s global -t ceph',
|
|
'interval': tri_hourly,
|
|
'max-sync-time': maxtime,
|
|
},
|
|
}
|
|
|
|
|
|
def mirror_status():
|
|
out = []
|
|
for x in repos:
|
|
repository = repos[x]
|
|
last_attempt = repository['last-attempt']
|
|
next_attempt = repository['last-attempt'] + repository['interval']
|
|
out.append( [x, last_attempt, next_attempt] )
|
|
|
|
out.sort(key= lambda x: x[2])
|
|
|
|
#turn floating point time values into human readable strings
|
|
for x in out:
|
|
for y in (1,2):
|
|
x[y] = time.ctime(x[y])
|
|
|
|
out.insert(0,['Repository', 'Last Synced', 'Next Expected Sync'])
|
|
|
|
#calculate maximum width of each column
|
|
widths = []
|
|
i = 0
|
|
while True:
|
|
try:
|
|
column_width = max([len(row[i]) for row in out])
|
|
widths.append(column_width + 3)
|
|
i = i + 1
|
|
except:
|
|
break
|
|
|
|
#compose table string and pad out columns
|
|
status_string = '%s%s%s\n' % (out[0][0].ljust(widths[0]), out[0][1].ljust(widths[1]), out[0][2].ljust(widths[2]))
|
|
for x in out[1:]:
|
|
for y in (0,1,2):
|
|
status_string = status_string + ('%s' % x[y].rjust(widths[y]))
|
|
status_string = status_string + '\n'
|
|
|
|
return status_string
|
|
|
|
|
|
def init_last_sync():
|
|
|
|
if not os.path.isdir('stamps'):
|
|
os.mkdir('stamps')
|
|
|
|
now = time.time()
|
|
|
|
for repo in repos:
|
|
try:
|
|
last = os.stat('stamps/%s' % repo).st_mtime
|
|
repos[repo]['last-sync'], repos[repo]['last-attempt'] = last, last
|
|
logging.info('repo %s last synced %d seconds ago' % (repo, now - last))
|
|
except OSError:
|
|
repos[repo]['last-sync'], repos[repo]['last-attempt'] = 0, 0
|
|
logging.warning('repo %s has never been synced' % repo)
|
|
|
|
|
|
def update_last_attempt(repo, start_time):
|
|
repos[repo]['last-attempt'] = start_time
|
|
|
|
def update_last_sync(repo, start_time, duration, exit_code):
|
|
|
|
repos[repo]['last-sync'] = start_time
|
|
update_last_attempt(repo, start_time)
|
|
|
|
# touch the timestamp file
|
|
with open('stamps/%s' % repo, 'w') as f:
|
|
f.write('%d,%d,%d' % (start_time, duration, exit_code))
|
|
# open('stamps/%s' % repo, 'w').close()
|
|
|
|
def handler(signum, frame):
|
|
logging.info("Caught signal %d" % signum)
|
|
|
|
os.remove(earPath)
|
|
|
|
for job in jobs:
|
|
os.kill(job, signal.SIGTERM)
|
|
|
|
time.sleep(1)
|
|
|
|
for job in jobs:
|
|
os.kill(job, signal.SIGKILL)
|
|
|
|
try:
|
|
while True:
|
|
pid, status = os.wait()
|
|
logging.info("Child process %d has terminated" % pid)
|
|
|
|
#ECHILD is thrown when there are no child processes left
|
|
except OSError, e:
|
|
if e.errno != errno.ECHILD:
|
|
raise
|
|
logging.info("All child processes terminated. Goodbye!")
|
|
sys.exit()
|
|
|
|
def setup_logging():
|
|
if not os.path.isdir('logs'):
|
|
os.mkdir('logs')
|
|
|
|
try:
|
|
os.unlink(earPath)
|
|
except OSError:
|
|
pass
|
|
|
|
if '-d' in sys.argv:
|
|
logging.basicConfig(filename="/home/mirror/merlin/logs/merlin.log",
|
|
level=logging.DEBUG, format="%(asctime)-15s %(message)s")
|
|
else:
|
|
logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s")
|
|
|
|
def sync(current, now):
|
|
pid = os.fork()
|
|
if not pid:
|
|
try:
|
|
logfd = os.open('logs/%s' % current, os.O_WRONLY|os.O_APPEND|os.O_CREAT, 0644)
|
|
nulfd = os.open('/dev/null', os.O_RDONLY)
|
|
os.dup2(nulfd, 0)
|
|
os.dup2(logfd, 1)
|
|
os.dup2(logfd, 2)
|
|
os.close(logfd)
|
|
os.close(nulfd)
|
|
os.execv("/bin/sh", ['sh', '-c', repos[current]['command']])
|
|
except OSError, e:
|
|
print >>sys.stderr, 'failed to exec: %s' % e
|
|
os.exit(1)
|
|
|
|
#There exists a race condition that manifests if merlin is asked to terminate ( by receiving signal ) after fork() but before the jobs table is updated.
|
|
#Normally you would mask off signals to avoid attempting shutdown in that time period, but this is not supported in the version of Python for which merlin was written.
|
|
#TODO:
|
|
#The Linux Programming Interface 24.5 illustrates how to synchronize parent and child using signals. Might be applicable.
|
|
|
|
jobs[pid] = {'name': current, 'start_time': now, 'status': 'running'}
|
|
|
|
def zfssync(current):
|
|
pid = os.fork()
|
|
if not pid:
|
|
try:
|
|
logfd = os.open('logs/zfssync-%s.log' % current, os.O_WRONLY|os.O_APPEND|os.O_CREAT, 0644)
|
|
nulfd = os.open('/dev/null', os.O_RDONLY)
|
|
os.dup2(nulfd, 0)
|
|
os.dup2(logfd, 1)
|
|
os.dup2(logfd, 2)
|
|
os.close(logfd)
|
|
os.close(nulfd)
|
|
os.execv("/home/mirror/bin/zfssync", ['zfssync', current])
|
|
except OSError, e:
|
|
print >>sys.stderr, 'failed to exec: %s' % e
|
|
os.exit(1)
|
|
|
|
def await_command(ear):
|
|
if select.select([ear],[],[],1) == ([ear],[],[]):
|
|
#handle command heard on ear socket
|
|
s, address = ear.accept()
|
|
cmdstring = ''
|
|
while True:
|
|
data = s.recv(cmd_buf_size)
|
|
if not data:
|
|
break
|
|
cmdstring = cmdstring + data
|
|
cmdstring = cmdstring.split(':',1)
|
|
command = cmdstring[0]
|
|
try:
|
|
if command == 'sync':
|
|
try:
|
|
arg = cmdstring[1]
|
|
if arg in repos:
|
|
if arg in (x['name'] for x in jobs.itervalues()):
|
|
logging.info('Cannot force sync: %s. Already syncing.' % arg)
|
|
s.send('Cannot force sync %s, already syncing.' % command)
|
|
else:
|
|
logging.info('Forcing sync: %s' % arg)
|
|
s.send('Forcing sync: %s' % arg)
|
|
sync(arg, time.time())
|
|
else:
|
|
logging.info('%s not tracked, cannot sync.' % arg)
|
|
s.send('%s not tracked, cannot sync.' % arg)
|
|
except:
|
|
s.send('Could not parse sync command, forced sync fails.')
|
|
raise
|
|
|
|
elif command == 'status':
|
|
s.send(mirror_status())
|
|
|
|
elif command == 'dump':
|
|
s.send(str(jobs))
|
|
s.send("\n")
|
|
s.send(str(repos))
|
|
|
|
else:
|
|
logging.error('Received unrecognized command: %s' % command)
|
|
s.send('Bad command: %s' % command)
|
|
|
|
s.close()
|
|
|
|
except socket.error, e:
|
|
logging.error('Could not communicate with arthur over socket.')
|
|
|
|
def new_jobs(now):
|
|
#To prevent repos at the 'bottom' of the dictionary from getting neglected when mirror is under unusual load (and merlin is running at MAX_JOBS)
|
|
keys = repos.keys()
|
|
random.shuffle(keys)
|
|
for current in keys:
|
|
if len(jobs) >= MAX_JOBS:
|
|
break
|
|
if now <= repos[current]['last-attempt'] + mintime:
|
|
continue
|
|
if current in (x['name'] for x in jobs.itervalues()):
|
|
continue
|
|
when_due = repos[current]['last-sync'] + repos[current]['interval']
|
|
if now >= when_due:
|
|
logging.debug("syncing %s, due for %d seconds" % (current, now - when_due))
|
|
sync(current, now)
|
|
|
|
def handle_completed(now):
|
|
try:
|
|
pid, status = os.waitpid(-1, os.WNOHANG) #waitpid() likes to throw an exception when there's nothing to wait for
|
|
|
|
#Don't know if we should be forever ignoring hung jobs if they eventually return...
|
|
if pid != 0 and pid in jobs:
|
|
job = jobs.pop(pid)
|
|
|
|
if os.WIFSIGNALED(status):
|
|
logging.error("%s %s sync terminated with signal %d" % (job['status'], job['name'], os.WTERMSIG(status)))
|
|
update_last_attempt(job['name'], job['start_time'])
|
|
elif os.WIFEXITED(status):
|
|
exit_status = os.WEXITSTATUS(status)
|
|
logging.error("%s %s sync exited with status %d " % (job['status'], job['name'], exit_status))
|
|
update_last_attempt(job['name'], job['start_time'])
|
|
interval = repos[job['name']]['interval']
|
|
sync_took = int(now - job['start_time'])
|
|
next_sync = max(0, interval - sync_took)
|
|
update_last_sync(job['name'], job['start_time'], sync_took, exit_status)
|
|
logging.info('%s sync complete, took %d seconds, syncs again in %d seconds'
|
|
% (job['name'], sync_took, next_sync))
|
|
|
|
zfssync(job['name'])
|
|
|
|
except OSError, e:
|
|
if e.errno != errno.ECHILD:
|
|
raise
|
|
|
|
def check_hung(now):
|
|
for pid in jobs:
|
|
#check if hung
|
|
runtime = now - jobs[pid]['start_time']
|
|
repo = jobs[pid]['name']
|
|
if jobs[pid]['status'] == 'running' and runtime > repos[repo]['max-sync-time']:
|
|
jobs[pid]['status'] = 'hung'
|
|
logging.error("%s sync process hung, process pid %d" % (repo, pid))
|
|
|
|
def main():
|
|
if not os.geteuid():
|
|
print "Don't run merlin as root!"
|
|
sys.exit(1);
|
|
|
|
setup_logging()
|
|
|
|
logging.info("Starting Merlin")
|
|
|
|
signal.signal(signal.SIGTERM, handler)
|
|
signal.signal(signal.SIGINT, handler)
|
|
|
|
init_last_sync()
|
|
|
|
|
|
old_umask = os.umask(0o002)
|
|
ear = socket.socket(socket.AF_UNIX)
|
|
ear.bind(earPath)
|
|
ear.listen(1)
|
|
os.umask(old_umask)
|
|
os.chown(earPath, -1, grp.getgrnam("push").gr_gid)
|
|
|
|
while True:
|
|
await_command(ear)
|
|
|
|
now = time.time()
|
|
|
|
new_jobs(now)
|
|
|
|
handle_completed(now)
|
|
|
|
check_hung(now)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|