resolve merge conflict

.drone.yml

@@ -12,6 +12,9 @@ steps:
     - go test ./...
 
 trigger:
+  event:
+    exclude:
+      - pull_request
   branch:
     - master
     - go

.gitignore

@@ -4,3 +4,12 @@
 /*.bz2
 /passwords/
 /tmp/
+/.bash_history
+/.lesshst
+/.vim/
+/.viminfo
+/.zcompdump
+/.aptitude/
+/.config/
+/.cache/
+/.ssh/

bin/csc-sync-ssh

@@ -14,11 +14,13 @@ SSH_USER=$4
 SSH_KEYFILE=$5
 RSYNC_HOST=$SSH_USER@$RSYNC_HOST
 
+ADDRESS=$(cat ~/config/ADDRESS)
+
 exec nice rsync -aH --no-owner --no-group --delete \
      --timeout=3600 -4 \
      --exclude .~tmp~/ \
      --quiet --stats --log-file=/home/mirror/merlin/logs/transfer-ssh.log \
-     -e "ssh -i $SSH_KEYFILE" \
+     -e "ssh -b $ADDRESS -i $SSH_KEYFILE" \
      $RSYNC_HOST:$RSYNC_DIR/ $TO
 
 #134.71 needs to be used

merlin_old/merlin.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python2
 
-import time, sys, os, errno, logging, signal, copy, select, socket, grp
+import time, sys, os, errno, logging, signal, copy, select, socket, grp, random
 
 daily = 86400
 twice_daily = 86400 / 2
@@ -192,11 +192,17 @@ repos = {
         'interval': five_minutely,
         'max-sync-time': maxtime,
     },
-    'debian-ports': {
+    'artixlinux': {
-        'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports',
+        'command': '~/bin/csc-sync-standard artixlinux mirror1.artixlinux.org artix-linux',
-        'interval': twice_daily,
+        'interval': bi_hourly,
         'max-sync-time': maxtime,
     },
+    # We stopped mirroring debian-ports on 2021-12-22 due to disk performance issues.
+    #'debian-ports': {
+    #    'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports',
+    #    'interval': twice_daily,
+    #    'max-sync-time': maxtime,
+    #},
     'slackware': {
         'command': '~/bin/csc-sync-standard slackware slackware.cs.utah.edu slackware',
         'interval': twice_daily,
@@ -210,7 +216,7 @@ repos = {
     'x.org': {
         #'command': '~/bin/csc-sync-standard x.org xorg.freedesktop.org xorg-archive',
         #'command': '~/bin/csc-sync-standard x.org mirror.us.leaseweb.net xorg',
-        'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub',
+        'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub/',
         'interval': twice_daily,
         'max-sync-time': maxtime,
     },
@@ -275,11 +281,12 @@ repos = {
         'interval': twice_daily,
         'max-sync-time': maxtime,
     },
-    'OpenBSD': {
+    # We stopped mirroring OpenBSD on 2021-12-22 due to degraded disk performance
-        'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp',
+    #'OpenBSD': {
-        'interval': twice_daily,
+    #    'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp',
-        'max-sync-time': maxtime,
+    #    'interval': twice_daily,
-    },
+    #    'max-sync-time': maxtime,
+    #},
     'xiph': {
         #'command': '~/bin/csc-sync-standard xiph downloads.xiph.org xiph/releases',
         'command': '~/bin/csc-sync-standard xiph ftp.osuosl.org xiph',
@@ -292,12 +299,13 @@ repos = {
         'interval': twice_daily,
         'max-sync-time': maxtime,
     },
-    'netbsd-pkgsrc': {
+    # We stopped mirroring pkgsrc on 2021-12-22 due to disk performance issues.
-        'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc',
+    #'netbsd-pkgsrc': {
-        #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/',
+    #    'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc',
-        'interval': twice_daily,
+    #    #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/',
-        'max-sync-time': maxtime,
+    #    'interval': twice_daily,
-    },
+    #    'max-sync-time': maxtime,
+    #},
     'macports-release': {
         'command': '~/bin/csc-sync-standard MacPorts/release rsync.macports.org macports/release/',
         'interval': bi_hourly,
@@ -329,11 +337,12 @@ repos = {
         'interval': twice_daily,
         'max-sync-time': maxtime,
     },
-    'qtproject': {
+    # We stopped mirroring qtproject on 2021-12-22 due to disk performance issues.
-        'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all',
+    #'qtproject': {
-        'interval': twice_daily,
+    #    'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all',
-        'max-sync-time': maxtime,
+    #    'interval': twice_daily,
-    },
+    #    'max-sync-time': maxtime,
+    #},
     'tdf': {
         'command': '~/bin/csc-sync-standard tdf rsync.documentfoundation.org tdf-pub',
         'interval': twice_daily,
@@ -360,7 +369,7 @@ repos = {
         'max-sync-time': maxtime,
     },
     'raspbian': {
-        'command': '~/bin/csc-sync-standard raspbian archive.raspbian.org archive',
+        'command': '~/bin/csc-sync-standard raspbian raspbian.freemirror.org raspbian',
         'interval': bi_hourly,
         'max-sync-time': maxtime,
     },
@@ -379,11 +388,6 @@ repos = {
         'interval': hourly,
         'max-sync-time': maxtime,
     },
-    'scientific': {
-        'command': '~/bin/csc-sync-standard scientific rsync.scientificlinux.org scientific',
-        'interval': bi_hourly,
-        'max-sync-time': maxtime,
-    },
     'mxlinux': {
         'command': '~/bin/csc-sync-standard mxlinux mirror.math.princeton.edu pub/mxlinux/',
         'interval': twice_daily,
@@ -619,6 +623,11 @@ def await_command(ear):
             elif command == 'status':
                 s.send(mirror_status())
 
+            elif command == 'dump':
+                s.send(str(jobs))
+                s.send("\n")
+                s.send(str(repos))
+
             else:
                 logging.error('Received unrecognized command: %s' % command)
                 s.send('Bad command: %s' % command)
@@ -629,7 +638,10 @@ def await_command(ear):
             logging.error('Could not communicate with arthur over socket.')
 
 def new_jobs(now):
-    for current in repos:
+    #To prevent repos at the 'bottom' of the dictionary from getting neglected when mirror is under unusual load (and merlin is running at MAX_JOBS)
+    keys = repos.keys()
+    random.shuffle(keys)
+    for current in keys:
         if len(jobs) >= MAX_JOBS:
             break
         if now <= repos[current]['last-attempt'] + mintime:

merlin_old/zfssync.yml

@@ -503,14 +503,14 @@ projects: {}
 ##      phys-1002-201.cloud.cs.uwaterloo.ca:
 ##        pool: cscmirror1
 ##        dataset: xubuntu-releases
-##  scientific:
+##  artixlinux:
 ##    hosts:
 ##      potassium-benzoate:
 ##        pool: cscmirror2
-##        dataset: scientific
+##        dataset: artixlinux
 ##      phys-1002-201.cloud.cs.uwaterloo.ca:
 ##        pool: cscmirror1
-##        dataset: scientific
+##        dataset: artixlinux
 ##  mxlinux:
 ##    hosts:
 ##      potassium-benzoate:

mirror-index/config.yaml

@@ -29,6 +29,10 @@ directories:
     site: archlinux.org
     url: https://www.archlinux.org/
 
+  artixlinux:
+    site: artixlinux.org
+    url: https://artixlinux.org/
+
   centos:
     site: centos.org
     url: https://www.centos.org/
@@ -77,10 +81,6 @@ directories:
     site: debian-multimedia.org
     url: https://www.debian-multimedia.org/
 
-  debian-ports:
-    site: debian-ports.org
-    url: https://www.debian-ports.org/
-
   debian-security:
     site: debian.org
     url: https://www.debian.org/security/
@@ -186,18 +186,10 @@ directories:
     site: opensuse.org
     url: https://www.opensuse.org/
 
-  OpenBSD:
-    site: openbsd.org
-    url: https://www.openbsd.org/
-
   puppylinux:
     site: puppylinux.org
     url: https://puppylinux.org/
 
-  qtproject:
-    site: qt-project.org
-    url: https://qt-project.org/
-
   racket:
     site: racket-lang.org
     url: https://racket-lang.org/
@@ -254,10 +246,6 @@ directories:
     site: netbsd.org
     url: https://www.netbsd.org/
 
-  pkgsrc:
-    site: pkgsrc.org
-    url: https://pkgsrc.org/
-
   raspberrypi:
     site: raspberrypi.org
     url: https://www.raspberrypi.org/
@@ -274,10 +262,6 @@ directories:
     site: manjaro.org
     url: https://manjaro.org/
 
-  scientific:
-    site: scientificlinux.org
-    url: https://www.scientificlinux.org/
-
   mxlinux:
     site: mxlinux.org
     url: https://mxlinux.org