From 6025e4804913b7609541fe863c76c1e4a8ba9f43 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 15 Jul 2022 04:31:43 +0000 Subject: [PATCH] finalize move and add report_mirror --- merlin/Handoff.md | 64 +++++++++++++++++------------------ merlin/README.md | 4 ++- merlin/config/config.go | 3 ++ merlin/merlin-config-test.ini | 29 ++++++++++++++++ merlin/merlin-config.ini | 57 +++++++++++++++++-------------- merlin/merlin-go.service | 2 +- merlin/sync/interface.go | 1 + merlin/sync/process.go | 12 +++++++ 8 files changed, 113 insertions(+), 59 deletions(-) create mode 100644 merlin/merlin-config-test.ini diff --git a/merlin/Handoff.md b/merlin/Handoff.md index 9b859d8..691a992 100644 --- a/merlin/Handoff.md +++ b/merlin/Handoff.md @@ -6,13 +6,13 @@ - [ ] debian-multimedia - [ ] debian-backports - [ ] debian-security -- [ ] ubuntu-releases (1 sec sync) -- [ ] xubuntu-releases (1 sec sync) +- [x] ubuntu-releases +- [x] xubuntu-releases - [ ] puppylinux (broken) - [ ] CPAN (unstable) - [ ] CRAN - [x] CTAN -- [ ] fedora-epel +- [ ] fedora-epel < - [x] cygwin - [x] gnu - [x] nongnu @@ -24,40 +24,40 @@ - [x] kernel.org-software - [x] apache - [ ] eclipse (unstable) -- [ ] kde < -- [ ] kde-appplicationdata < +- [x] kde +- [x] kde-applicationdata - [ ] archlinux -- [ ] artixlinx < -- [ ] slackware < +- [x] artixlinx +- [x] slackware - [ ] debian-cd -- [ ] x.org < -- [ ] centos < -- [ ] opensuse -- [ ] FreeBSD < -- [ ] fedora-enchilada -- [ ] ubuntu-ports-releases < +- [x] x.org +- [x] centos +- [ ] opensuse < +- [x] FreeBSD +- [ ] fedora-enchilada < +- [x] ubuntu-ports-releases - [ ] gentoo-distfiles -- [ ] gentoo-portage < -- [ ] gutenberg < +- [x] gentoo-portage +- [ ] gutenberg (broken) - [ ] racket-installers -- [ ] plt-bundles < -- [ ] xiph < -- [ ] netbsd < -- [ ] macports-release < -- [ ] macports-distfiles < -- [ ] sagemath < -- [ ] vlc < -- [ ] tdf < +- [ ] plt-bundles (broken) +- [x] xiph +- [ ] netbsd (broken) +- [x] macports-release +- [x] macports-distfiles +- [x] sagemath +- [x] vlc +- [x] tdf - [ ] saltstack -- [ ] alpine < -- [ ] raspbian < +- [x] alpine +- [ ] raspbian (high disk usage + long runtime + failing) - [ ] raspberrypi - [ ] ipfire (broken) -- [ ] manjaro < -- [ ] mxlinux < -- [ ] mxlinux-iso < -- [ ] parabola < -- [ ] trisquel-packages < -- [ ] trisquel-iso < -- [ ] almalinux < +- [x] manjaro +- [x] mxlinux +- [x] mxlinux-iso +- [x] parabola +- [x] trisquel-packages +- [x] trisquel-iso +- [x] almalinux - [ ] ceph diff --git a/merlin/README.md b/merlin/README.md index 552479e..67c68f0 100644 --- a/merlin/README.md +++ b/merlin/README.md @@ -16,7 +16,9 @@ Then configure `merlin-config.ini` and run using `./merlin` - get the config or the rsync commond that a repo will sync with using a cli tool - improve conversion from exit status enum to string - sort `arthur status` by last time synced -- time spent to sync during last sync, seconds/minutes/hours until next sync +- last sync runtime, time until next sync +- when sync fails, then make a copy of the rsync logs +- create `csc-sync-fedora` rather than using the `report_mirror` setting ### Completed - [x] add bwlimit option for each rsync process diff --git a/merlin/config/config.go b/merlin/config/config.go index 4b7931a..6bc1fff 100644 --- a/merlin/config/config.go +++ b/merlin/config/config.go @@ -132,6 +132,8 @@ type Repo struct { RsyncLogFile string `ini:"-"` // full file path for file logging this repo's zfssync ZfssyncLogFile string `ini:"-"` + // determines if report_mirror should be run after the sync (default: false) + ReportMirror bool `ini:"report_mirror"` // add the "-vv" flag to rsync commands and enable the Debug log (default: false) Verbose bool `ini:"verbose"` // the repo will write its name and status in a Result struct to DoneChan @@ -232,6 +234,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc RepoLogFile: filepath.Join(newConf.RepoLogDir, repoName) + ".log", RsyncLogFile: filepath.Join(newConf.RsyncLogDir, repoName) + "-rsync.log", ZfssyncLogFile: filepath.Join(newConf.ZfssyncLogDir, repoName) + "-zfssync.log", + ReportMirror: false, Verbose: false, DoneChan: doneChan, StopChan: stopChan, diff --git a/merlin/merlin-config-test.ini b/merlin/merlin-config-test.ini new file mode 100644 index 0000000..cde8f8e --- /dev/null +++ b/merlin/merlin-config-test.ini @@ -0,0 +1,29 @@ +ipv4_address = 129.97.134.71 +ipv6_address = 2620:101:f000:4901:c5c::f:1055 + +; max_jobs = 6 + +download_dir = /mirror/root +trace_dir = /home/mirror-go/merlin/trace +state_dir = /home/mirror-go/merlin/state + +repo_log_dir = /home/mirror-go/merlin/log +rsync_log_dir = /home/mirror-go/merlin/log-rsync +zfssync_log_dir = /home/mirror-go/merlin/log-zfssync + +sock_path = /mirror/merlin/run/merlin-go.sock + +[ubuntu] +dry_run = true +sync_type = csc-sync-standard +frequency = five-minutely +local_dir = ubuntu +rsync_host = rsync.releases.ubuntu.com +rsync_dir = releases +rsync_exclude = distribution/.timestamp_invisiable, .ignore_me + +; add to the repo struct +; RsyncExclude []string `ini:"rsync_exclude"` + +; then iterate over RsyncExclude: +; for _, exclude := range repo.RsyncExclude {} \ No newline at end of file diff --git a/merlin/merlin-config.ini b/merlin/merlin-config.ini index 59f8fdf..9034232 100644 --- a/merlin/merlin-config.ini +++ b/merlin/merlin-config.ini @@ -14,7 +14,6 @@ zfssync_log_dir = /home/mirror-go/merlin/log-zfssync sock_path = /mirror/merlin/run/merlin-go.sock [ubuntu-releases] -verbose = true sync_type = csc-sync-standard frequency = bi-hourly local_dir = ubuntu-releases @@ -22,7 +21,6 @@ rsync_host = rsync.releases.ubuntu.com rsync_dir = releases [xubuntu-releases] -verbose = true sync_type = csc-sync-standard frequency = bi-hourly local_dir = xubuntu-releases @@ -52,6 +50,16 @@ local_dir = CTAN rsync_host = rsync.dante.ctan.org rsync_dir = CTAN +[fedora-epel] +dry_run = true +verbose = true +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = fedora/epel +rsync_host = mirrors.kernel.org +rsync_dir = fedora-epel +report_mirror = true + [cygwin] sync_type = csc-sync-standard frequency = twice-daily @@ -133,7 +141,6 @@ rsync_host = download.eclipse.org rsync_dir = eclipseMirror [kde] -verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = kde @@ -141,7 +148,6 @@ rsync_host = rsync.kde.org rsync_dir = kdeftp [kde-applicationdata] -verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = kde-applicationdata @@ -149,7 +155,6 @@ rsync_host = rsync.kde.org rsync_dir = applicationdata [artixlinux] -verbose = true sync_type = csc-sync-standard frequency = bi-hourly local_dir = artixlinux @@ -157,7 +162,6 @@ rsync_host = mirror1.artixlinux.org rsync_dir = artix-linux [slackware] -verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = slackware @@ -165,7 +169,6 @@ rsync_host = slackware.cs.utah.edu rsync_dir = slackware [x.org] -verbose = true ; csc-sync-standard x.org xorg.freedesktop.org xorg-archive ; csc-sync-standard x.org mirror.us.leaseweb.net xorg sync_type = csc-sync-standard @@ -175,15 +178,24 @@ rsync_host = rsync.mirrorservice.org rsync_dir = ftp.x.org/pub/ [centos] -verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = centos rsync_host = us-msync.centos.org rsync_dir = CentOS -[FreeBSD] +[opensuse] +dry_run = true verbose = true +; unsure what this setting is supposed to do +; #"--exclude distribution/.timestamp_invisible" +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = opensuse +rsync_host = stage.opensuse.org +rsync_dir = opensuse-full/opensuse/ + +[FreeBSD] ; Has not updated since at least June 2018 ; csc-sync-standard FreeBSD ftp10.us.freebsd.org FreeBSD sync_type = csc-sync-standard @@ -193,8 +205,18 @@ local_dir = FreeBSD rsync_host = ftp2.uk.freebsd.org rsync_dir = ftp.freebsd.org/pub/FreeBSD/ -[ubuntu-ports-releases] +[fedora-enchilada] +dry_run = true verbose = true +; csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = fedora/linux +rsync_host = mirrors.kernel.org +rsync_dir = fedora-enchilada/linux/ +report_mirror = true + +[ubuntu-ports-releases] sync_type = csc-sync-standard frequency = twice-daily local_dir = ubuntu-ports-releases @@ -202,7 +224,6 @@ rsync_host = cdimage.ubuntu.com rsync_dir = cdimage/releases/ [gentoo-portage] -verbose = true sync_type = csc-sync-standard frequency = twice-hourly local_dir = gentoo-portage @@ -226,7 +247,6 @@ rsync_host = mirror.racket-lang.org rsync_dir = plt-bundles [xiph] -verbose = true ; csc-sync-standard xiph downloads.xiph.org xiph/releases sync_type = csc-sync-standard frequency = twice-daily @@ -244,7 +264,6 @@ rsync_host = rsync.netbsd.org rsync_dir = NetBSD [macports-release] -verbose = true sync_type = csc-sync-standard frequency = bi-hourly local_dir = MacPorts/release @@ -252,7 +271,6 @@ rsync_host = rsync.macports.org rsync_dir = macports/release/ [macports-distfiles] -verbose = true sync_type = csc-sync-standard frequency = bi-hourly local_dir = MacPorts/mpdistfiles @@ -260,7 +278,6 @@ rsync_host = rsync.macports.org rsync_dir = macports/distfiles/ [sagemath] -verbose = true ; csc-sync-standard sage mirror.clibre.uqam.ca sage sync_type = csc-sync-standard frequency = twice-daily @@ -269,7 +286,6 @@ rsync_host = rsync.sagemath.org rsync_dir = sage [vlc] -verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = vlc @@ -277,7 +293,6 @@ rsync_host = rsync.videolan.org rsync_dir = videolan-ftp [tdf] -verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = tdf @@ -285,7 +300,6 @@ rsync_host = rsync.documentfoundation.org rsync_dir = tdf-pub [alpine] -verbose = true sync_type = csc-sync-standard frequency = hourly local_dir = alpine @@ -309,7 +323,6 @@ rsync_host = rsync.ipfire.org rsync_dir = full [manjaro] -verbose = true sync_type = csc-sync-standard frequency = hourly local_dir = manjaro @@ -317,7 +330,6 @@ rsync_host = mirrorservice.org rsync_dir = repo.manjaro.org/repos/ [mxlinux] -verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = mxlinux @@ -325,7 +337,6 @@ rsync_host = mirror.math.princeton.edu rsync_dir = pub/mxlinux/ [mxlinux-iso] -verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = mxlinux-iso @@ -333,7 +344,6 @@ rsync_host = mirror.math.princeton.edu rsync_dir = pub/mxlinux-iso/ [parabola] -verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = parabola @@ -341,7 +351,6 @@ rsync_host = repo.parabola.nu:875 rsync_dir = repos/ [trisquel-packages] -verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = trisquel/packages @@ -349,7 +358,6 @@ rsync_host = rsync.trisquel.info rsync_dir = trisquel.packages/ [trisquel-iso] -verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = trisquel/iso @@ -357,7 +365,6 @@ rsync_host = rsync.trisquel.info rsync_dir = trisquel.iso/ [almalinux] -verbose = true sync_type = csc-sync-standard frequency = bi-hourly local_dir = almalinux diff --git a/merlin/merlin-go.service b/merlin/merlin-go.service index 03971a0..12e0731 100644 --- a/merlin/merlin-go.service +++ b/merlin/merlin-go.service @@ -3,7 +3,7 @@ Description=Manages synchronization of mirrored projects After=network.target [Service] -ExecStart=/home/mirror-go/merlin/merlin /home/mirror-go/merlin/merlin-config.ini +ExecStart=/home/mirror-go/merlin/merlin --config=/home/mirror-go/merlin/merlin-config.ini WorkingDirectory=/home/mirror-go/merlin User=mirror Group=mirror diff --git a/merlin/sync/interface.go b/merlin/sync/interface.go index 3a88357..c404523 100644 --- a/merlin/sync/interface.go +++ b/merlin/sync/interface.go @@ -50,5 +50,6 @@ func SyncCompleted(repo *config.Repo, exit int) { if exit == config.SUCCESS { go zfsSync(repo) + go reportMirror(repo) } } diff --git a/merlin/sync/process.go b/merlin/sync/process.go index 7838d15..867568c 100644 --- a/merlin/sync/process.go +++ b/merlin/sync/process.go @@ -146,3 +146,15 @@ func zfsSync(repo *config.Repo) { } } } + +func reportMirror(repo *config.Repo) { + if !repo.ReportMirror { + return + } + + repo.Logger.Debug("Running report_mirror") + err := exec.Command("/home/mirror/bin/report_mirror").Run() + if err != nil { + repo.Logger.Error(fmt.Errorf("Error running report_mirror: %w", err)) + } +}