resolve merge conflict

1 year ago · 72c143a00b
parent 363073f818 d68741a21d
commit 72c143a00b
6 changed files with 62 additions and 52 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -12,6 +12,9 @@ steps:
    - go test ./...

 trigger:
+  event:
+    exclude:
+      - pull_request
  branch:
    - master
    - go
--- a/.gitignore
+++ b/.gitignore
@ -4,3 +4,12 @@
 /*.bz2
 /passwords/
 /tmp/
+/.bash_history
+/.lesshst
+/.vim/
+/.viminfo
+/.zcompdump
+/.aptitude/
+/.config/
+/.cache/
+/.ssh/
--- a/bin/csc-sync-ssh
+++ b/bin/csc-sync-ssh
@ -14,11 +14,13 @@ SSH_USER=$4
 SSH_KEYFILE=$5
 RSYNC_HOST=$SSH_USER@$RSYNC_HOST

+ADDRESS=$(cat ~/config/ADDRESS)
+
 exec nice rsync -aH --no-owner --no-group --delete \
     --timeout=3600 -4 \
     --exclude .~tmp~/ \
     --quiet --stats --log-file=/home/mirror/merlin/logs/transfer-ssh.log \
-     -e "ssh -i $SSH_KEYFILE" \
+     -e "ssh -b $ADDRESS -i $SSH_KEYFILE" \
     $RSYNC_HOST:$RSYNC_DIR/ $TO

 #134.71 needs to be used
--- a/merlin_old/merlin.py
+++ b/merlin_old/merlin.py
@ -1,6 +1,6 @@
 #!/usr/bin/python2

-import time, sys, os, errno, logging, signal, copy, select, socket, grp
+import time, sys, os, errno, logging, signal, copy, select, socket, grp, random

 daily = 86400
 twice_daily = 86400 / 2
@ -192,11 +192,17 @@ repos = {
        'interval': five_minutely,
        'max-sync-time': maxtime,
    },
-    'debian-ports': {
-        'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports',
-        'interval': twice_daily,
+    'artixlinux': {
+        'command': '~/bin/csc-sync-standard artixlinux mirror1.artixlinux.org artix-linux',
+        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
+    # We stopped mirroring debian-ports on 2021-12-22 due to disk performance issues.
+    #'debian-ports': {
+    #    'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports',
+    #    'interval': twice_daily,
+    #    'max-sync-time': maxtime,
+    #},
    'slackware': {
        'command': '~/bin/csc-sync-standard slackware slackware.cs.utah.edu slackware',
        'interval': twice_daily,
@ -210,7 +216,7 @@ repos = {
    'x.org': {
        #'command': '~/bin/csc-sync-standard x.org xorg.freedesktop.org xorg-archive',
        #'command': '~/bin/csc-sync-standard x.org mirror.us.leaseweb.net xorg',
-        'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub',
+        'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub/',
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
@ -275,11 +281,12 @@ repos = {
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
-    'OpenBSD': {
-        'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp',
-        'interval': twice_daily,
-        'max-sync-time': maxtime,
-    },
+    # We stopped mirroring OpenBSD on 2021-12-22 due to degraded disk performance
+    #'OpenBSD': {
+    #    'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp',
+    #    'interval': twice_daily,
+    #    'max-sync-time': maxtime,
+    #},
    'xiph': {
        #'command': '~/bin/csc-sync-standard xiph downloads.xiph.org xiph/releases',
        'command': '~/bin/csc-sync-standard xiph ftp.osuosl.org xiph',
@ -292,12 +299,13 @@ repos = {
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
-    'netbsd-pkgsrc': {
-        'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc',
-        #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/',
-        'interval': twice_daily,
-        'max-sync-time': maxtime,
-    },
+    # We stopped mirroring pkgsrc on 2021-12-22 due to disk performance issues.
+    #'netbsd-pkgsrc': {
+    #    'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc',
+    #    #'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/',
+    #    'interval': twice_daily,
+    #    'max-sync-time': maxtime,
+    #},
    'macports-release': {
        'command': '~/bin/csc-sync-standard MacPorts/release rsync.macports.org macports/release/',
        'interval': bi_hourly,
@ -329,11 +337,12 @@ repos = {
        'interval': twice_daily,
        'max-sync-time': maxtime,
    },
-    'qtproject': {
-        'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all',
-        'interval': twice_daily,
-        'max-sync-time': maxtime,
-    },
+    # We stopped mirroring qtproject on 2021-12-22 due to disk performance issues.
+    #'qtproject': {
+    #    'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all',
+    #    'interval': twice_daily,
+    #    'max-sync-time': maxtime,
+    #},
    'tdf': {
        'command': '~/bin/csc-sync-standard tdf rsync.documentfoundation.org tdf-pub',
        'interval': twice_daily,
@ -360,7 +369,7 @@ repos = {
        'max-sync-time': maxtime,
    },
    'raspbian': {
-        'command': '~/bin/csc-sync-standard raspbian archive.raspbian.org archive',
+        'command': '~/bin/csc-sync-standard raspbian raspbian.freemirror.org raspbian',
        'interval': bi_hourly,
        'max-sync-time': maxtime,
    },
@ -379,11 +388,6 @@ repos = {
        'interval': hourly,
        'max-sync-time': maxtime,
    },
-    'scientific': {
-        'command': '~/bin/csc-sync-standard scientific rsync.scientificlinux.org scientific',
-        'interval': bi_hourly,
-        'max-sync-time': maxtime,
-    },
    'mxlinux': {
        'command': '~/bin/csc-sync-standard mxlinux mirror.math.princeton.edu pub/mxlinux/',
        'interval': twice_daily,
@ -619,6 +623,11 @@ def await_command(ear):
            elif command == 'status':
                s.send(mirror_status())

+            elif command == 'dump':
+                s.send(str(jobs))
+                s.send("\n")
+                s.send(str(repos))
+
            else:
                logging.error('Received unrecognized command: %s' % command)
                s.send('Bad command: %s' % command)
@ -629,7 +638,10 @@ def await_command(ear):
            logging.error('Could not communicate with arthur over socket.')

 def new_jobs(now):
-    for current in repos:
+    #To prevent repos at the 'bottom' of the dictionary from getting neglected when mirror is under unusual load (and merlin is running at MAX_JOBS)
+    keys = repos.keys()
+    random.shuffle(keys)
+    for current in keys:
        if len(jobs) >= MAX_JOBS:
            break
        if now <= repos[current]['last-attempt'] + mintime:
--- a/merlin_old/zfssync.yml
+++ b/merlin_old/zfssync.yml
@ -503,14 +503,14 @@ projects: {}
 ##      phys-1002-201.cloud.cs.uwaterloo.ca:
 ##        pool: cscmirror1
 ##        dataset: xubuntu-releases
-##  scientific:
+##  artixlinux:
 ##    hosts:
 ##      potassium-benzoate:
 ##        pool: cscmirror2
-##        dataset: scientific
+##        dataset: artixlinux
 ##      phys-1002-201.cloud.cs.uwaterloo.ca:
 ##        pool: cscmirror1
-##        dataset: scientific
+##        dataset: artixlinux
 ##  mxlinux:
 ##    hosts:
 ##      potassium-benzoate:
--- a/mirror-index/config.yaml
+++ b/mirror-index/config.yaml
@ -29,6 +29,10 @@ directories:
    site: archlinux.org
    url: https://www.archlinux.org/

+  artixlinux:
+    site: artixlinux.org
+    url: https://artixlinux.org/
+
  centos:
    site: centos.org
    url: https://www.centos.org/
@ -77,10 +81,6 @@ directories:
    site: debian-multimedia.org
    url: https://www.debian-multimedia.org/

-  debian-ports:
-    site: debian-ports.org
-    url: https://www.debian-ports.org/
-
  debian-security:
    site: debian.org
    url: https://www.debian.org/security/
@ -186,18 +186,10 @@ directories:
    site: opensuse.org
    url: https://www.opensuse.org/

-  OpenBSD:
-    site: openbsd.org
-    url: https://www.openbsd.org/
-
  puppylinux:
    site: puppylinux.org
    url: https://puppylinux.org/

-  qtproject:
-    site: qt-project.org
-    url: https://qt-project.org/
-
  racket:
    site: racket-lang.org
    url: https://racket-lang.org/
@ -254,10 +246,6 @@ directories:
    site: netbsd.org
    url: https://www.netbsd.org/

-  pkgsrc:
-    site: pkgsrc.org
-    url: https://pkgsrc.org/
-
  raspberrypi:
    site: raspberrypi.org
    url: https://www.raspberrypi.org/
@ -274,10 +262,6 @@ directories:
    site: manjaro.org
    url: https://manjaro.org/

-  scientific:
-    site: scientificlinux.org
-    url: https://www.scientificlinux.org/
-
  mxlinux:
    site: mxlinux.org
    url: https://mxlinux.org