resolve merge conflict

This commit is contained in:
Andrew Wang 2022-02-11 23:27:58 -05:00
commit 72c143a00b
6 changed files with 62 additions and 52 deletions

View File

@ -12,6 +12,9 @@ steps:
- go test ./...
trigger:
event:
exclude:
- pull_request
branch:
- master
- go

9
.gitignore vendored
View File

@ -4,3 +4,12 @@
/*.bz2
/passwords/
/tmp/
/.bash_history
/.lesshst
/.vim/
/.viminfo
/.zcompdump
/.aptitude/
/.config/
/.cache/
/.ssh/

View File

@ -14,11 +14,13 @@ SSH_USER=$4
SSH_KEYFILE=$5
RSYNC_HOST=$SSH_USER@$RSYNC_HOST
ADDRESS=$(cat ~/config/ADDRESS)
exec nice rsync -aH --no-owner --no-group --delete \
--timeout=3600 -4 \
--exclude .~tmp~/ \
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer-ssh.log \
-e "ssh -i $SSH_KEYFILE" \
-e "ssh -b $ADDRESS -i $SSH_KEYFILE" \
$RSYNC_HOST:$RSYNC_DIR/ $TO
#134.71 needs to be used

View File

@ -1,6 +1,6 @@
#!/usr/bin/python2
import time, sys, os, errno, logging, signal, copy, select, socket, grp
import time, sys, os, errno, logging, signal, copy, select, socket, grp, random
daily = 86400
twice_daily = 86400 / 2
@ -192,11 +192,17 @@ repos = {
'interval': five_minutely,
'max-sync-time': maxtime,
},
'debian-ports': {
'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports',
'interval': twice_daily,
'artixlinux': {
'command': '~/bin/csc-sync-standard artixlinux mirror1.artixlinux.org artix-linux',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
# We stopped mirroring debian-ports on 2021-12-22 due to disk performance issues.
#'debian-ports': {
# 'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
#},
'slackware': {
'command': '~/bin/csc-sync-standard slackware slackware.cs.utah.edu slackware',
'interval': twice_daily,
@ -210,7 +216,7 @@ repos = {
'x.org': {
#'command': '~/bin/csc-sync-standard x.org xorg.freedesktop.org xorg-archive',
#'command': '~/bin/csc-sync-standard x.org mirror.us.leaseweb.net xorg',
'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub',
'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
@ -275,11 +281,12 @@ repos = {
'interval': twice_daily,
'max-sync-time': maxtime,
},
'OpenBSD': {
'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp',
'interval': twice_daily,
'max-sync-time': maxtime,
},
# We stopped mirroring OpenBSD on 2021-12-22 due to degraded disk performance
#'OpenBSD': {
# 'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
#},
'xiph': {
#'command': '~/bin/csc-sync-standard xiph downloads.xiph.org xiph/releases',
'command': '~/bin/csc-sync-standard xiph ftp.osuosl.org xiph',
@ -292,12 +299,13 @@ repos = {
'interval': twice_daily,
'max-sync-time': maxtime,
},
'netbsd-pkgsrc': {
'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc',
#'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/',
'interval': twice_daily,
'max-sync-time': maxtime,
},
# We stopped mirroring pkgsrc on 2021-12-22 due to disk performance issues.
#'netbsd-pkgsrc': {
# 'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc',
# #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
#},
'macports-release': {
'command': '~/bin/csc-sync-standard MacPorts/release rsync.macports.org macports/release/',
'interval': bi_hourly,
@ -329,11 +337,12 @@ repos = {
'interval': twice_daily,
'max-sync-time': maxtime,
},
'qtproject': {
'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all',
'interval': twice_daily,
'max-sync-time': maxtime,
},
# We stopped mirroring qtproject on 2021-12-22 due to disk performance issues.
#'qtproject': {
# 'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all',
# 'interval': twice_daily,
# 'max-sync-time': maxtime,
#},
'tdf': {
'command': '~/bin/csc-sync-standard tdf rsync.documentfoundation.org tdf-pub',
'interval': twice_daily,
@ -360,7 +369,7 @@ repos = {
'max-sync-time': maxtime,
},
'raspbian': {
'command': '~/bin/csc-sync-standard raspbian archive.raspbian.org archive',
'command': '~/bin/csc-sync-standard raspbian raspbian.freemirror.org raspbian',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
@ -379,11 +388,6 @@ repos = {
'interval': hourly,
'max-sync-time': maxtime,
},
'scientific': {
'command': '~/bin/csc-sync-standard scientific rsync.scientificlinux.org scientific',
'interval': bi_hourly,
'max-sync-time': maxtime,
},
'mxlinux': {
'command': '~/bin/csc-sync-standard mxlinux mirror.math.princeton.edu pub/mxlinux/',
'interval': twice_daily,
@ -619,6 +623,11 @@ def await_command(ear):
elif command == 'status':
s.send(mirror_status())
elif command == 'dump':
s.send(str(jobs))
s.send("\n")
s.send(str(repos))
else:
logging.error('Received unrecognized command: %s' % command)
s.send('Bad command: %s' % command)
@ -629,7 +638,10 @@ def await_command(ear):
logging.error('Could not communicate with arthur over socket.')
def new_jobs(now):
for current in repos:
#To prevent repos at the 'bottom' of the dictionary from getting neglected when mirror is under unusual load (and merlin is running at MAX_JOBS)
keys = repos.keys()
random.shuffle(keys)
for current in keys:
if len(jobs) >= MAX_JOBS:
break
if now <= repos[current]['last-attempt'] + mintime:

View File

@ -503,14 +503,14 @@ projects: {}
## phys-1002-201.cloud.cs.uwaterloo.ca:
## pool: cscmirror1
## dataset: xubuntu-releases
## scientific:
## artixlinux:
## hosts:
## potassium-benzoate:
## pool: cscmirror2
## dataset: scientific
## dataset: artixlinux
## phys-1002-201.cloud.cs.uwaterloo.ca:
## pool: cscmirror1
## dataset: scientific
## dataset: artixlinux
## mxlinux:
## hosts:
## potassium-benzoate:

View File

@ -29,6 +29,10 @@ directories:
site: archlinux.org
url: https://www.archlinux.org/
artixlinux:
site: artixlinux.org
url: https://artixlinux.org/
centos:
site: centos.org
url: https://www.centos.org/
@ -77,10 +81,6 @@ directories:
site: debian-multimedia.org
url: https://www.debian-multimedia.org/
debian-ports:
site: debian-ports.org
url: https://www.debian-ports.org/
debian-security:
site: debian.org
url: https://www.debian.org/security/
@ -186,18 +186,10 @@ directories:
site: opensuse.org
url: https://www.opensuse.org/
OpenBSD:
site: openbsd.org
url: https://www.openbsd.org/
puppylinux:
site: puppylinux.org
url: https://puppylinux.org/
qtproject:
site: qt-project.org
url: https://qt-project.org/
racket:
site: racket-lang.org
url: https://racket-lang.org/
@ -254,10 +246,6 @@ directories:
site: netbsd.org
url: https://www.netbsd.org/
pkgsrc:
site: pkgsrc.org
url: https://pkgsrc.org/
raspberrypi:
site: raspberrypi.org
url: https://www.raspberrypi.org/
@ -274,10 +262,6 @@ directories:
site: manjaro.org
url: https://manjaro.org/
scientific:
site: scientificlinux.org
url: https://www.scientificlinux.org/
mxlinux:
site: mxlinux.org
url: https://mxlinux.org