From 9c37804f757083b258458e5992572478a0b668b7 Mon Sep 17 00:00:00 2001 From: Anthony Brennan Date: Tue, 14 Sep 2021 18:51:17 -0400 Subject: [PATCH 01/15] Add a handy command for dumping internal data over a socket for inspection --- merlin/merlin.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/merlin/merlin.py b/merlin/merlin.py index e2f603c..cd6fb06 100755 --- a/merlin/merlin.py +++ b/merlin/merlin.py @@ -619,6 +619,11 @@ def await_command(ear): elif command == 'status': s.send(mirror_status()) + elif command == 'dump': + s.send(str(jobs)) + s.send("\n") + s.send(str(repos)) + else: logging.error('Received unrecognized command: %s' % command) s.send('Bad command: %s' % command) From 4f4f1a52af05872120af43a754d06bc4acfba3e6 Mon Sep 17 00:00:00 2001 From: Anthony Brennan Date: Tue, 14 Sep 2021 19:00:18 -0400 Subject: [PATCH 02/15] Prevent problem where repos at the 'bottom' of the dictionary were failing to sync during periods of extended load --- merlin/merlin.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/merlin/merlin.py b/merlin/merlin.py index cd6fb06..0099a6e 100755 --- a/merlin/merlin.py +++ b/merlin/merlin.py @@ -1,6 +1,6 @@ #!/usr/bin/python2 -import time, sys, os, errno, logging, signal, copy, select, socket, grp +import time, sys, os, errno, logging, signal, copy, select, socket, grp, random daily = 86400 twice_daily = 86400 / 2 @@ -634,7 +634,10 @@ def await_command(ear): logging.error('Could not communicate with arthur over socket.') def new_jobs(now): - for current in repos: + #To prevent repos at the 'bottom' of the dictionary from getting neglected when mirror is under unusual load (and merlin is running at MAX_JOBS) + keys = repos.keys() + random.shuffle(keys) + for current in keys: if len(jobs) >= MAX_JOBS: break if now <= repos[current]['last-attempt'] + mintime: From 9181264ad574717263144ab013597dbc01bb5f19 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Fri, 15 Oct 2021 22:41:16 -0400 Subject: [PATCH 03/15] add trailing slash to x.org remote folder --- merlin/merlin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merlin/merlin.py b/merlin/merlin.py index 0099a6e..2447834 100755 --- a/merlin/merlin.py +++ b/merlin/merlin.py @@ -210,7 +210,7 @@ repos = { 'x.org': { #'command': '~/bin/csc-sync-standard x.org xorg.freedesktop.org xorg-archive', #'command': '~/bin/csc-sync-standard x.org mirror.us.leaseweb.net xorg', - 'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub', + 'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub/', 'interval': twice_daily, 'max-sync-time': maxtime, }, From 162910c81c5b1cf63f1f7c272da6f9880ad3cb5b Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Sun, 17 Oct 2021 15:21:15 -0400 Subject: [PATCH 04/15] use raspbian.freemirror.org --- merlin/merlin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merlin/merlin.py b/merlin/merlin.py index 2447834..eae0302 100755 --- a/merlin/merlin.py +++ b/merlin/merlin.py @@ -360,7 +360,7 @@ repos = { 'max-sync-time': maxtime, }, 'raspbian': { - 'command': '~/bin/csc-sync-standard raspbian archive.raspbian.org archive', + 'command': '~/bin/csc-sync-standard raspbian raspbian.freemirror.org raspbian', 'interval': bi_hourly, 'max-sync-time': maxtime, }, From 03ec30013a90c866935c11a7ce1ee36f491813cf Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Thu, 18 Nov 2021 02:50:39 -0500 Subject: [PATCH 05/15] remove scientific and add artixlinux --- merlin/merlin.py | 10 +++++----- merlin/zfssync.yml | 6 +++--- mirror-index/config.yaml | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/merlin/merlin.py b/merlin/merlin.py index eae0302..8183048 100755 --- a/merlin/merlin.py +++ b/merlin/merlin.py @@ -192,6 +192,11 @@ repos = { 'interval': five_minutely, 'max-sync-time': maxtime, }, + 'artixlinux': { + 'command': '~/bin/csc-sync-standard artixlinux mirror1.artixlinux.org artix-linux', + 'interval': bi_hourly, + 'max-sync-time': maxtime, + }, 'debian-ports': { 'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports', 'interval': twice_daily, @@ -379,11 +384,6 @@ repos = { 'interval': hourly, 'max-sync-time': maxtime, }, - 'scientific': { - 'command': '~/bin/csc-sync-standard scientific rsync.scientificlinux.org scientific', - 'interval': bi_hourly, - 'max-sync-time': maxtime, - }, 'mxlinux': { 'command': '~/bin/csc-sync-standard mxlinux mirror.math.princeton.edu pub/mxlinux/', 'interval': twice_daily, diff --git a/merlin/zfssync.yml b/merlin/zfssync.yml index dc863cc..3532ae0 100644 --- a/merlin/zfssync.yml +++ b/merlin/zfssync.yml @@ -503,14 +503,14 @@ projects: {} ## phys-1002-201.cloud.cs.uwaterloo.ca: ## pool: cscmirror1 ## dataset: xubuntu-releases -## scientific: +## artixlinux: ## hosts: ## potassium-benzoate: ## pool: cscmirror2 -## dataset: scientific +## dataset: artixlinux ## phys-1002-201.cloud.cs.uwaterloo.ca: ## pool: cscmirror1 -## dataset: scientific +## dataset: artixlinux ## mxlinux: ## hosts: ## potassium-benzoate: diff --git a/mirror-index/config.yaml b/mirror-index/config.yaml index edd06d2..b7fe111 100644 --- a/mirror-index/config.yaml +++ b/mirror-index/config.yaml @@ -29,6 +29,10 @@ directories: site: archlinux.org url: https://www.archlinux.org/ + artixlinux: + site: artixlinux.org + url: https://artixlinux.org/ + centos: site: centos.org url: https://www.centos.org/ @@ -274,10 +278,6 @@ directories: site: manjaro.org url: https://manjaro.org/ - scientific: - site: scientificlinux.org - url: https://www.scientificlinux.org/ - mxlinux: site: mxlinux.org url: https://mxlinux.org From 38964d353faa59d35551d2a9634e794248bd4b74 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Thu, 18 Nov 2021 02:54:03 -0500 Subject: [PATCH 06/15] add modified SSH sync script --- bin/csc-sync-ssh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/csc-sync-ssh b/bin/csc-sync-ssh index 1db4a5a..23a863a 100755 --- a/bin/csc-sync-ssh +++ b/bin/csc-sync-ssh @@ -14,11 +14,13 @@ SSH_USER=$4 SSH_KEYFILE=$5 RSYNC_HOST=$SSH_USER@$RSYNC_HOST +ADDRESS=$(cat ~/config/ADDRESS) + exec nice rsync -aH --no-owner --no-group --delete \ --timeout=3600 -4 \ --exclude .~tmp~/ \ --quiet --stats --log-file=/home/mirror/merlin/logs/transfer-ssh.log \ - -e "ssh -i $SSH_KEYFILE" \ + -e "ssh -b $ADDRESS -i $SSH_KEYFILE" \ $RSYNC_HOST:$RSYNC_DIR/ $TO #134.71 needs to be used From 0d9240527282f175b3f29d414cbe1d3b8239e80e Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 15 Dec 2021 01:32:48 -0500 Subject: [PATCH 07/15] add .drone.yml --- .drone.yml | 17 +++++++++++++++++ .gitignore | 1 - 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..755f90b --- /dev/null +++ b/.drone.yml @@ -0,0 +1,17 @@ +--- +kind: pipeline +type: docker +name: default + +steps: +- name: merlin + image: golang:1.17 + commands: + # add linter + - cd merlin + - go build + - go test + +trigger: + event: + - push diff --git a/.gitignore b/.gitignore index f9b3e14..c746cbd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -/.* !.gitignore .git_old/ /dead.letter From 0be12a4b3fe3745090afc3395db7fd2bc9c48fd5 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 15 Dec 2021 17:33:10 -0500 Subject: [PATCH 08/15] modify .drone.yml --- .drone.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 755f90b..f6fa5c4 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,4 +1,3 @@ ---- kind: pipeline type: docker name: default @@ -13,5 +12,7 @@ steps: - go test trigger: - event: - - push + branch: + - master + - go + - refactor From f588a4acf4741161f813dbee87730a0a74b0de29 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 15 Dec 2021 18:01:28 -0500 Subject: [PATCH 09/15] update .drone.yml --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index f6fa5c4..0d02039 100644 --- a/.drone.yml +++ b/.drone.yml @@ -9,7 +9,7 @@ steps: # add linter - cd merlin - go build - - go test + - go test ./... trigger: branch: From cc67e7c1b452f262c9dd8000e6aaa988605676af Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 15 Dec 2021 22:33:11 -0500 Subject: [PATCH 10/15] disable double builds on PR --- .drone.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.drone.yml b/.drone.yml index 0d02039..6d7401c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -12,6 +12,9 @@ steps: - go test ./... trigger: + event: + exclude: + - pull_request branch: - master - go From 610ee8d0ab833f544d8c1b674217cfc9fae2fc62 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Wed, 22 Dec 2021 12:21:37 -0500 Subject: [PATCH 11/15] remove OpenBSD --- merlin/merlin.py | 11 ++++++----- mirror-index/config.yaml | 4 ---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/merlin/merlin.py b/merlin/merlin.py index 8183048..b7397b9 100755 --- a/merlin/merlin.py +++ b/merlin/merlin.py @@ -280,11 +280,12 @@ repos = { 'interval': twice_daily, 'max-sync-time': maxtime, }, - 'OpenBSD': { - 'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp', - 'interval': twice_daily, - 'max-sync-time': maxtime, - }, + # We stopped mirroring OpenBSD on 2021-12-22 due to degraded disk performance + #'OpenBSD': { + # 'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp', + # 'interval': twice_daily, + # 'max-sync-time': maxtime, + #}, 'xiph': { #'command': '~/bin/csc-sync-standard xiph downloads.xiph.org xiph/releases', 'command': '~/bin/csc-sync-standard xiph ftp.osuosl.org xiph', diff --git a/mirror-index/config.yaml b/mirror-index/config.yaml index b7fe111..e51532a 100644 --- a/mirror-index/config.yaml +++ b/mirror-index/config.yaml @@ -190,10 +190,6 @@ directories: site: opensuse.org url: https://www.opensuse.org/ - OpenBSD: - site: openbsd.org - url: https://www.openbsd.org/ - puppylinux: site: puppylinux.org url: https://puppylinux.org/ From d33d7c3cc0c613ee6f9ac74188b555f634e98934 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Wed, 22 Dec 2021 12:27:40 -0500 Subject: [PATCH 12/15] update .gitignore --- .gitignore | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitignore b/.gitignore index c746cbd..c012532 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,12 @@ /*.bz2 /passwords/ /tmp/ +/.bash_history +/.lesshst +/.vim/ +/.viminfo +/.zcompdump +/.aptitude/ +/.config/ +/.cache/ +/.ssh/ From 47d21a3f056a52dd7c952c2c8c44742af763ac31 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Wed, 22 Dec 2021 12:33:20 -0500 Subject: [PATCH 13/15] remove debian-ports --- merlin/merlin.py | 11 ++++++----- mirror-index/config.yaml | 4 ---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/merlin/merlin.py b/merlin/merlin.py index b7397b9..b75d743 100755 --- a/merlin/merlin.py +++ b/merlin/merlin.py @@ -197,11 +197,12 @@ repos = { 'interval': bi_hourly, 'max-sync-time': maxtime, }, - 'debian-ports': { - 'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports', - 'interval': twice_daily, - 'max-sync-time': maxtime, - }, + # We stopped mirroring debian-ports on 2021-12-22 due to disk performance issues. + #'debian-ports': { + # 'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports', + # 'interval': twice_daily, + # 'max-sync-time': maxtime, + #}, 'slackware': { 'command': '~/bin/csc-sync-standard slackware slackware.cs.utah.edu slackware', 'interval': twice_daily, diff --git a/mirror-index/config.yaml b/mirror-index/config.yaml index e51532a..47acc36 100644 --- a/mirror-index/config.yaml +++ b/mirror-index/config.yaml @@ -81,10 +81,6 @@ directories: site: debian-multimedia.org url: https://www.debian-multimedia.org/ - debian-ports: - site: debian-ports.org - url: https://www.debian-ports.org/ - debian-security: site: debian.org url: https://www.debian.org/security/ From 5036a665d39093ef928759b96b39b73feea4bd87 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Wed, 22 Dec 2021 12:54:56 -0500 Subject: [PATCH 14/15] remove qtproject --- merlin/merlin.py | 11 ++++++----- mirror-index/config.yaml | 4 ---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/merlin/merlin.py b/merlin/merlin.py index b75d743..fe07596 100755 --- a/merlin/merlin.py +++ b/merlin/merlin.py @@ -336,11 +336,12 @@ repos = { 'interval': twice_daily, 'max-sync-time': maxtime, }, - 'qtproject': { - 'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all', - 'interval': twice_daily, - 'max-sync-time': maxtime, - }, + # We stopped mirroring qtproject on 2021-12-22 due to disk performance issues. + #'qtproject': { + # 'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all', + # 'interval': twice_daily, + # 'max-sync-time': maxtime, + #}, 'tdf': { 'command': '~/bin/csc-sync-standard tdf rsync.documentfoundation.org tdf-pub', 'interval': twice_daily, diff --git a/mirror-index/config.yaml b/mirror-index/config.yaml index 47acc36..d112356 100644 --- a/mirror-index/config.yaml +++ b/mirror-index/config.yaml @@ -190,10 +190,6 @@ directories: site: puppylinux.org url: https://puppylinux.org/ - qtproject: - site: qt-project.org - url: https://qt-project.org/ - racket: site: racket-lang.org url: https://racket-lang.org/ From d68741a21dcc7e52f8b220ce97a1c096699b3b7c Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Wed, 22 Dec 2021 13:07:02 -0500 Subject: [PATCH 15/15] remove pkgsrc --- merlin/merlin.py | 13 +++++++------ mirror-index/config.yaml | 4 ---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/merlin/merlin.py b/merlin/merlin.py index fe07596..b77de73 100755 --- a/merlin/merlin.py +++ b/merlin/merlin.py @@ -299,12 +299,13 @@ repos = { 'interval': twice_daily, 'max-sync-time': maxtime, }, - 'netbsd-pkgsrc': { - 'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc', - #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/', - 'interval': twice_daily, - 'max-sync-time': maxtime, - }, + # We stopped mirroring pkgsrc on 2021-12-22 due to disk performance issues. + #'netbsd-pkgsrc': { + # 'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc', + # #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/', + # 'interval': twice_daily, + # 'max-sync-time': maxtime, + #}, 'macports-release': { 'command': '~/bin/csc-sync-standard MacPorts/release rsync.macports.org macports/release/', 'interval': bi_hourly, diff --git a/mirror-index/config.yaml b/mirror-index/config.yaml index d112356..101e344 100644 --- a/mirror-index/config.yaml +++ b/mirror-index/config.yaml @@ -246,10 +246,6 @@ directories: site: netbsd.org url: https://www.netbsd.org/ - pkgsrc: - site: pkgsrc.org - url: https://pkgsrc.org/ - raspberrypi: site: raspberrypi.org url: https://www.raspberrypi.org/