finalize move and add report_mirror

merlin/Handoff.md

@@ -6,13 +6,13 @@
 - [ ] debian-multimedia
 - [ ] debian-backports
 - [ ] debian-security
-- [ ] ubuntu-releases (1 sec sync)
-- [ ] xubuntu-releases (1 sec sync)
+- [x] ubuntu-releases
+- [x] xubuntu-releases
 - [ ] puppylinux (broken)
 - [ ] CPAN (unstable)
 - [ ] CRAN
 - [x] CTAN
-- [ ] fedora-epel
+- [ ] fedora-epel <
 - [x] cygwin
 - [x] gnu
 - [x] nongnu
@@ -24,40 +24,40 @@
 - [x] kernel.org-software
 - [x] apache
 - [ ] eclipse (unstable)
-- [ ] kde <
-- [ ] kde-appplicationdata <
+- [x] kde
+- [x] kde-applicationdata
 - [ ] archlinux
-- [ ] artixlinx <
-- [ ] slackware <
+- [x] artixlinx
+- [x] slackware
 - [ ] debian-cd
-- [ ] x.org <
-- [ ] centos <
-- [ ] opensuse
-- [ ] FreeBSD <
-- [ ] fedora-enchilada
-- [ ] ubuntu-ports-releases <
+- [x] x.org
+- [x] centos
+- [ ] opensuse <
+- [x] FreeBSD
+- [ ] fedora-enchilada <
+- [x] ubuntu-ports-releases
 - [ ] gentoo-distfiles
-- [ ] gentoo-portage <
-- [ ] gutenberg <
+- [x] gentoo-portage
+- [ ] gutenberg (broken)
 - [ ] racket-installers
-- [ ] plt-bundles <
-- [ ] xiph <
-- [ ] netbsd <
-- [ ] macports-release <
-- [ ] macports-distfiles <
-- [ ] sagemath <
-- [ ] vlc <
-- [ ] tdf <
+- [ ] plt-bundles (broken)
+- [x] xiph
+- [ ] netbsd (broken)
+- [x] macports-release
+- [x] macports-distfiles
+- [x] sagemath
+- [x] vlc
+- [x] tdf
 - [ ] saltstack
-- [ ] alpine <
-- [ ] raspbian <
+- [x] alpine
+- [ ] raspbian (high disk usage + long runtime + failing)
 - [ ] raspberrypi
 - [ ] ipfire (broken)
-- [ ] manjaro <
-- [ ] mxlinux <
-- [ ] mxlinux-iso <
-- [ ] parabola <
-- [ ] trisquel-packages <
-- [ ] trisquel-iso <
-- [ ] almalinux <
+- [x] manjaro
+- [x] mxlinux
+- [x] mxlinux-iso
+- [x] parabola
+- [x] trisquel-packages
+- [x] trisquel-iso
+- [x] almalinux
 - [ ] ceph

merlin/README.md

@@ -16,7 +16,9 @@ Then configure `merlin-config.ini` and run using `./merlin`
 - get the config or the rsync commond that a repo will sync with using a cli tool
 - improve conversion from exit status enum to string
 - sort `arthur status` by last time synced
-- time spent to sync during last sync, seconds/minutes/hours until next sync
+- last sync runtime, time until next sync
+- when sync fails, then make a copy of the rsync logs
+- create `csc-sync-fedora` rather than using the `report_mirror` setting
 
 ### Completed
 - [x] add bwlimit option for each rsync process

merlin/config/config.go

@@ -132,6 +132,8 @@ type Repo struct {
 	RsyncLogFile string `ini:"-"`
 	// full file path for file logging this repo's zfssync
 	ZfssyncLogFile string `ini:"-"`
+	// determines if report_mirror should be run after the sync (default: false)
+	ReportMirror bool `ini:"report_mirror"`
 	// add the "-vv" flag to rsync commands and enable the Debug log (default: false)
 	Verbose bool `ini:"verbose"`
 	// the repo will write its name and status in a Result struct to DoneChan
@@ -232,6 +234,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
 			RepoLogFile:    filepath.Join(newConf.RepoLogDir, repoName) + ".log",
 			RsyncLogFile:   filepath.Join(newConf.RsyncLogDir, repoName) + "-rsync.log",
 			ZfssyncLogFile: filepath.Join(newConf.ZfssyncLogDir, repoName) + "-zfssync.log",
+			ReportMirror:   false,
 			Verbose:        false,
 			DoneChan:       doneChan,
 			StopChan:       stopChan,

merlin/merlin-config-test.ini

@@ -0,0 +1,29 @@
+ipv4_address = 129.97.134.71
+ipv6_address = 2620:101:f000:4901:c5c::f:1055
+
+; max_jobs = 6
+
+download_dir = /mirror/root
+trace_dir = /home/mirror-go/merlin/trace
+state_dir = /home/mirror-go/merlin/state
+
+repo_log_dir = /home/mirror-go/merlin/log
+rsync_log_dir = /home/mirror-go/merlin/log-rsync
+zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
+
+sock_path = /mirror/merlin/run/merlin-go.sock
+
+[ubuntu]
+dry_run = true
+sync_type = csc-sync-standard
+frequency = five-minutely
+local_dir = ubuntu
+rsync_host = rsync.releases.ubuntu.com
+rsync_dir = releases
+rsync_exclude = distribution/.timestamp_invisiable, .ignore_me
+
+; add to the repo struct
+; RsyncExclude []string `ini:"rsync_exclude"`
+
+; then iterate over RsyncExclude:
+; for _, exclude := range repo.RsyncExclude {}

merlin/merlin-config.ini

@@ -14,7 +14,6 @@ zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
 sock_path = /mirror/merlin/run/merlin-go.sock
 
 [ubuntu-releases]
-verbose = true
 sync_type = csc-sync-standard
 frequency = bi-hourly
 local_dir = ubuntu-releases
@@ -22,7 +21,6 @@ rsync_host = rsync.releases.ubuntu.com
 rsync_dir = releases
 
 [xubuntu-releases]
-verbose = true
 sync_type = csc-sync-standard
 frequency = bi-hourly
 local_dir = xubuntu-releases
@@ -52,6 +50,16 @@ local_dir = CTAN
 rsync_host = rsync.dante.ctan.org
 rsync_dir = CTAN
 
+[fedora-epel]
+dry_run = true
+verbose = true
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = fedora/epel
+rsync_host = mirrors.kernel.org
+rsync_dir = fedora-epel
+report_mirror = true
+
 [cygwin]
 sync_type = csc-sync-standard
 frequency = twice-daily
@@ -133,7 +141,6 @@ rsync_host = download.eclipse.org
 rsync_dir = eclipseMirror
 
 [kde]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = kde
@@ -141,7 +148,6 @@ rsync_host = rsync.kde.org
 rsync_dir = kdeftp
 
 [kde-applicationdata]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = kde-applicationdata
@@ -149,7 +155,6 @@ rsync_host = rsync.kde.org
 rsync_dir = applicationdata
 
 [artixlinux]
-verbose = true
 sync_type = csc-sync-standard
 frequency = bi-hourly
 local_dir = artixlinux
@@ -157,7 +162,6 @@ rsync_host = mirror1.artixlinux.org
 rsync_dir = artix-linux
 
 [slackware]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = slackware
@@ -165,7 +169,6 @@ rsync_host = slackware.cs.utah.edu
 rsync_dir = slackware
 
 [x.org]
-verbose = true
 ; csc-sync-standard x.org xorg.freedesktop.org xorg-archive
 ; csc-sync-standard x.org mirror.us.leaseweb.net xorg
 sync_type = csc-sync-standard
@@ -175,15 +178,24 @@ rsync_host = rsync.mirrorservice.org
 rsync_dir = ftp.x.org/pub/
 
 [centos]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = centos
 rsync_host = us-msync.centos.org
 rsync_dir = CentOS
 
-[FreeBSD]
+[opensuse]
+dry_run = true
 verbose = true
+; unsure what this setting is supposed to do
+; #"--exclude distribution/.timestamp_invisible"
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = opensuse
+rsync_host = stage.opensuse.org
+rsync_dir = opensuse-full/opensuse/
+
+[FreeBSD]
 ; Has not updated since at least June 2018
 ; csc-sync-standard FreeBSD ftp10.us.freebsd.org FreeBSD
 sync_type = csc-sync-standard
@@ -193,8 +205,18 @@ local_dir = FreeBSD
 rsync_host = ftp2.uk.freebsd.org
 rsync_dir = ftp.freebsd.org/pub/FreeBSD/
 
-[ubuntu-ports-releases]
+[fedora-enchilada]
+dry_run = true
 verbose = true
+; csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = fedora/linux
+rsync_host = mirrors.kernel.org
+rsync_dir = fedora-enchilada/linux/
+report_mirror = true
+
+[ubuntu-ports-releases]
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = ubuntu-ports-releases
@@ -202,7 +224,6 @@ rsync_host = cdimage.ubuntu.com
 rsync_dir = cdimage/releases/
 
 [gentoo-portage]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-hourly
 local_dir = gentoo-portage
@@ -226,7 +247,6 @@ rsync_host = mirror.racket-lang.org
 rsync_dir = plt-bundles
 
 [xiph]
-verbose = true
 ; csc-sync-standard xiph downloads.xiph.org xiph/releases
 sync_type = csc-sync-standard
 frequency = twice-daily
@@ -244,7 +264,6 @@ rsync_host = rsync.netbsd.org
 rsync_dir = NetBSD
 
 [macports-release]
-verbose = true
 sync_type = csc-sync-standard
 frequency = bi-hourly
 local_dir = MacPorts/release
@@ -252,7 +271,6 @@ rsync_host = rsync.macports.org
 rsync_dir = macports/release/
 
 [macports-distfiles]
-verbose = true
 sync_type = csc-sync-standard
 frequency = bi-hourly
 local_dir = MacPorts/mpdistfiles
@@ -260,7 +278,6 @@ rsync_host = rsync.macports.org
 rsync_dir = macports/distfiles/
 
 [sagemath]
-verbose = true
 ; csc-sync-standard sage mirror.clibre.uqam.ca sage
 sync_type = csc-sync-standard
 frequency = twice-daily
@@ -269,7 +286,6 @@ rsync_host = rsync.sagemath.org
 rsync_dir = sage
 
 [vlc]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = vlc
@@ -277,7 +293,6 @@ rsync_host = rsync.videolan.org
 rsync_dir = videolan-ftp
 
 [tdf]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = tdf
@@ -285,7 +300,6 @@ rsync_host = rsync.documentfoundation.org
 rsync_dir = tdf-pub
 
 [alpine]
-verbose = true
 sync_type = csc-sync-standard
 frequency = hourly
 local_dir = alpine
@@ -309,7 +323,6 @@ rsync_host = rsync.ipfire.org
 rsync_dir = full
 
 [manjaro]
-verbose = true
 sync_type = csc-sync-standard
 frequency = hourly
 local_dir = manjaro
@@ -317,7 +330,6 @@ rsync_host = mirrorservice.org
 rsync_dir = repo.manjaro.org/repos/
 
 [mxlinux]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = mxlinux
@@ -325,7 +337,6 @@ rsync_host = mirror.math.princeton.edu
 rsync_dir = pub/mxlinux/
 
 [mxlinux-iso]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = mxlinux-iso
@@ -333,7 +344,6 @@ rsync_host = mirror.math.princeton.edu
 rsync_dir = pub/mxlinux-iso/
 
 [parabola]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = parabola
@@ -341,7 +351,6 @@ rsync_host = repo.parabola.nu:875
 rsync_dir = repos/
 
 [trisquel-packages]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = trisquel/packages
@@ -349,7 +358,6 @@ rsync_host = rsync.trisquel.info
 rsync_dir = trisquel.packages/
 
 [trisquel-iso]
-verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = trisquel/iso
@@ -357,7 +365,6 @@ rsync_host = rsync.trisquel.info
 rsync_dir = trisquel.iso/
 
 [almalinux]
-verbose = true
 sync_type = csc-sync-standard
 frequency = bi-hourly
 local_dir = almalinux

merlin/merlin-go.service

@@ -3,7 +3,7 @@ Description=Manages synchronization of mirrored projects
 After=network.target
 
 [Service]
-ExecStart=/home/mirror-go/merlin/merlin /home/mirror-go/merlin/merlin-config.ini
+ExecStart=/home/mirror-go/merlin/merlin --config=/home/mirror-go/merlin/merlin-config.ini
 WorkingDirectory=/home/mirror-go/merlin
 User=mirror
 Group=mirror

merlin/sync/interface.go

@@ -50,5 +50,6 @@ func SyncCompleted(repo *config.Repo, exit int) {
 
 	if exit == config.SUCCESS {
 		go zfsSync(repo)
+		go reportMirror(repo)
 	}
 }

merlin/sync/process.go

@@ -146,3 +146,15 @@ func zfsSync(repo *config.Repo) {
 		}
 	}
 }
+
+func reportMirror(repo *config.Repo) {
+	if !repo.ReportMirror {
+		return
+	}
+
+	repo.Logger.Debug("Running report_mirror")
+	err := exec.Command("/home/mirror/bin/report_mirror").Run()
+	if err != nil {
+		repo.Logger.Error(fmt.Errorf("Error running report_mirror: %w", err))
+	}
+}