diff --git a/.drone.yml b/.drone.yml index 0d02039..6d7401c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -12,6 +12,9 @@ steps: - go test ./... trigger: + event: + exclude: + - pull_request branch: - master - go diff --git a/.gitignore b/.gitignore index c746cbd..c012532 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,12 @@ /*.bz2 /passwords/ /tmp/ +/.bash_history +/.lesshst +/.vim/ +/.viminfo +/.zcompdump +/.aptitude/ +/.config/ +/.cache/ +/.ssh/ diff --git a/bin/csc-sync-ssh b/bin/csc-sync-ssh index 1db4a5a..23a863a 100755 --- a/bin/csc-sync-ssh +++ b/bin/csc-sync-ssh @@ -14,11 +14,13 @@ SSH_USER=$4 SSH_KEYFILE=$5 RSYNC_HOST=$SSH_USER@$RSYNC_HOST +ADDRESS=$(cat ~/config/ADDRESS) + exec nice rsync -aH --no-owner --no-group --delete \ --timeout=3600 -4 \ --exclude .~tmp~/ \ --quiet --stats --log-file=/home/mirror/merlin/logs/transfer-ssh.log \ - -e "ssh -i $SSH_KEYFILE" \ + -e "ssh -b $ADDRESS -i $SSH_KEYFILE" \ $RSYNC_HOST:$RSYNC_DIR/ $TO #134.71 needs to be used diff --git a/merlin_old/merlin.py b/merlin_old/merlin.py index e2f603c..b77de73 100755 --- a/merlin_old/merlin.py +++ b/merlin_old/merlin.py @@ -1,6 +1,6 @@ #!/usr/bin/python2 -import time, sys, os, errno, logging, signal, copy, select, socket, grp +import time, sys, os, errno, logging, signal, copy, select, socket, grp, random daily = 86400 twice_daily = 86400 / 2 @@ -192,11 +192,17 @@ repos = { 'interval': five_minutely, 'max-sync-time': maxtime, }, - 'debian-ports': { - 'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports', - 'interval': twice_daily, + 'artixlinux': { + 'command': '~/bin/csc-sync-standard artixlinux mirror1.artixlinux.org artix-linux', + 'interval': bi_hourly, 'max-sync-time': maxtime, }, + # We stopped mirroring debian-ports on 2021-12-22 due to disk performance issues. + #'debian-ports': { + # 'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports', + # 'interval': twice_daily, + # 'max-sync-time': maxtime, + #}, 'slackware': { 'command': '~/bin/csc-sync-standard slackware slackware.cs.utah.edu slackware', 'interval': twice_daily, @@ -210,7 +216,7 @@ repos = { 'x.org': { #'command': '~/bin/csc-sync-standard x.org xorg.freedesktop.org xorg-archive', #'command': '~/bin/csc-sync-standard x.org mirror.us.leaseweb.net xorg', - 'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub', + 'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub/', 'interval': twice_daily, 'max-sync-time': maxtime, }, @@ -275,11 +281,12 @@ repos = { 'interval': twice_daily, 'max-sync-time': maxtime, }, - 'OpenBSD': { - 'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp', - 'interval': twice_daily, - 'max-sync-time': maxtime, - }, + # We stopped mirroring OpenBSD on 2021-12-22 due to degraded disk performance + #'OpenBSD': { + # 'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp', + # 'interval': twice_daily, + # 'max-sync-time': maxtime, + #}, 'xiph': { #'command': '~/bin/csc-sync-standard xiph downloads.xiph.org xiph/releases', 'command': '~/bin/csc-sync-standard xiph ftp.osuosl.org xiph', @@ -292,12 +299,13 @@ repos = { 'interval': twice_daily, 'max-sync-time': maxtime, }, - 'netbsd-pkgsrc': { - 'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc', - #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/', - 'interval': twice_daily, - 'max-sync-time': maxtime, - }, + # We stopped mirroring pkgsrc on 2021-12-22 due to disk performance issues. + #'netbsd-pkgsrc': { + # 'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc', + # #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/', + # 'interval': twice_daily, + # 'max-sync-time': maxtime, + #}, 'macports-release': { 'command': '~/bin/csc-sync-standard MacPorts/release rsync.macports.org macports/release/', 'interval': bi_hourly, @@ -329,11 +337,12 @@ repos = { 'interval': twice_daily, 'max-sync-time': maxtime, }, - 'qtproject': { - 'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all', - 'interval': twice_daily, - 'max-sync-time': maxtime, - }, + # We stopped mirroring qtproject on 2021-12-22 due to disk performance issues. + #'qtproject': { + # 'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all', + # 'interval': twice_daily, + # 'max-sync-time': maxtime, + #}, 'tdf': { 'command': '~/bin/csc-sync-standard tdf rsync.documentfoundation.org tdf-pub', 'interval': twice_daily, @@ -360,7 +369,7 @@ repos = { 'max-sync-time': maxtime, }, 'raspbian': { - 'command': '~/bin/csc-sync-standard raspbian archive.raspbian.org archive', + 'command': '~/bin/csc-sync-standard raspbian raspbian.freemirror.org raspbian', 'interval': bi_hourly, 'max-sync-time': maxtime, }, @@ -379,11 +388,6 @@ repos = { 'interval': hourly, 'max-sync-time': maxtime, }, - 'scientific': { - 'command': '~/bin/csc-sync-standard scientific rsync.scientificlinux.org scientific', - 'interval': bi_hourly, - 'max-sync-time': maxtime, - }, 'mxlinux': { 'command': '~/bin/csc-sync-standard mxlinux mirror.math.princeton.edu pub/mxlinux/', 'interval': twice_daily, @@ -619,6 +623,11 @@ def await_command(ear): elif command == 'status': s.send(mirror_status()) + elif command == 'dump': + s.send(str(jobs)) + s.send("\n") + s.send(str(repos)) + else: logging.error('Received unrecognized command: %s' % command) s.send('Bad command: %s' % command) @@ -629,7 +638,10 @@ def await_command(ear): logging.error('Could not communicate with arthur over socket.') def new_jobs(now): - for current in repos: + #To prevent repos at the 'bottom' of the dictionary from getting neglected when mirror is under unusual load (and merlin is running at MAX_JOBS) + keys = repos.keys() + random.shuffle(keys) + for current in keys: if len(jobs) >= MAX_JOBS: break if now <= repos[current]['last-attempt'] + mintime: diff --git a/merlin_old/zfssync.yml b/merlin_old/zfssync.yml index dc863cc..3532ae0 100644 --- a/merlin_old/zfssync.yml +++ b/merlin_old/zfssync.yml @@ -503,14 +503,14 @@ projects: {} ## phys-1002-201.cloud.cs.uwaterloo.ca: ## pool: cscmirror1 ## dataset: xubuntu-releases -## scientific: +## artixlinux: ## hosts: ## potassium-benzoate: ## pool: cscmirror2 -## dataset: scientific +## dataset: artixlinux ## phys-1002-201.cloud.cs.uwaterloo.ca: ## pool: cscmirror1 -## dataset: scientific +## dataset: artixlinux ## mxlinux: ## hosts: ## potassium-benzoate: diff --git a/mirror-index/config.yaml b/mirror-index/config.yaml index edd06d2..101e344 100644 --- a/mirror-index/config.yaml +++ b/mirror-index/config.yaml @@ -29,6 +29,10 @@ directories: site: archlinux.org url: https://www.archlinux.org/ + artixlinux: + site: artixlinux.org + url: https://artixlinux.org/ + centos: site: centos.org url: https://www.centos.org/ @@ -77,10 +81,6 @@ directories: site: debian-multimedia.org url: https://www.debian-multimedia.org/ - debian-ports: - site: debian-ports.org - url: https://www.debian-ports.org/ - debian-security: site: debian.org url: https://www.debian.org/security/ @@ -186,18 +186,10 @@ directories: site: opensuse.org url: https://www.opensuse.org/ - OpenBSD: - site: openbsd.org - url: https://www.openbsd.org/ - puppylinux: site: puppylinux.org url: https://puppylinux.org/ - qtproject: - site: qt-project.org - url: https://qt-project.org/ - racket: site: racket-lang.org url: https://racket-lang.org/ @@ -254,10 +246,6 @@ directories: site: netbsd.org url: https://www.netbsd.org/ - pkgsrc: - site: pkgsrc.org - url: https://pkgsrc.org/ - raspberrypi: site: raspberrypi.org url: https://www.raspberrypi.org/ @@ -274,10 +262,6 @@ directories: site: manjaro.org url: https://manjaro.org/ - scientific: - site: scientificlinux.org - url: https://www.scientificlinux.org/ - mxlinux: site: mxlinux.org url: https://mxlinux.org