finalize move and add report_mirror

pull/3/head
Andrew Wang 5 months ago
parent c93e504c58
commit 6025e48049
  1. 64
      merlin/Handoff.md
  2. 4
      merlin/README.md
  3. 3
      merlin/config/config.go
  4. 29
      merlin/merlin-config-test.ini
  5. 57
      merlin/merlin-config.ini
  6. 2
      merlin/merlin-go.service
  7. 1
      merlin/sync/interface.go
  8. 12
      merlin/sync/process.go

@ -6,13 +6,13 @@
- [ ] debian-multimedia
- [ ] debian-backports
- [ ] debian-security
- [ ] ubuntu-releases (1 sec sync)
- [ ] xubuntu-releases (1 sec sync)
- [x] ubuntu-releases
- [x] xubuntu-releases
- [ ] puppylinux (broken)
- [ ] CPAN (unstable)
- [ ] CRAN
- [x] CTAN
- [ ] fedora-epel
- [ ] fedora-epel <
- [x] cygwin
- [x] gnu
- [x] nongnu
@ -24,40 +24,40 @@
- [x] kernel.org-software
- [x] apache
- [ ] eclipse (unstable)
- [ ] kde <
- [ ] kde-appplicationdata <
- [x] kde
- [x] kde-applicationdata
- [ ] archlinux
- [ ] artixlinx <
- [ ] slackware <
- [x] artixlinx
- [x] slackware
- [ ] debian-cd
- [ ] x.org <
- [ ] centos <
- [ ] opensuse
- [ ] FreeBSD <
- [ ] fedora-enchilada
- [ ] ubuntu-ports-releases <
- [x] x.org
- [x] centos
- [ ] opensuse <
- [x] FreeBSD
- [ ] fedora-enchilada <
- [x] ubuntu-ports-releases
- [ ] gentoo-distfiles
- [ ] gentoo-portage <
- [ ] gutenberg <
- [x] gentoo-portage
- [ ] gutenberg (broken)
- [ ] racket-installers
- [ ] plt-bundles <
- [ ] xiph <
- [ ] netbsd <
- [ ] macports-release <
- [ ] macports-distfiles <
- [ ] sagemath <
- [ ] vlc <
- [ ] tdf <
- [ ] plt-bundles (broken)
- [x] xiph
- [ ] netbsd (broken)
- [x] macports-release
- [x] macports-distfiles
- [x] sagemath
- [x] vlc
- [x] tdf
- [ ] saltstack
- [ ] alpine <
- [ ] raspbian <
- [x] alpine
- [ ] raspbian (high disk usage + long runtime + failing)
- [ ] raspberrypi
- [ ] ipfire (broken)
- [ ] manjaro <
- [ ] mxlinux <
- [ ] mxlinux-iso <
- [ ] parabola <
- [ ] trisquel-packages <
- [ ] trisquel-iso <
- [ ] almalinux <
- [x] manjaro
- [x] mxlinux
- [x] mxlinux-iso
- [x] parabola
- [x] trisquel-packages
- [x] trisquel-iso
- [x] almalinux
- [ ] ceph

@ -16,7 +16,9 @@ Then configure `merlin-config.ini` and run using `./merlin`
- get the config or the rsync commond that a repo will sync with using a cli tool
- improve conversion from exit status enum to string
- sort `arthur status` by last time synced
- time spent to sync during last sync, seconds/minutes/hours until next sync
- last sync runtime, time until next sync
- when sync fails, then make a copy of the rsync logs
- create `csc-sync-fedora` rather than using the `report_mirror` setting
### Completed
- [x] add bwlimit option for each rsync process

@ -132,6 +132,8 @@ type Repo struct {
RsyncLogFile string `ini:"-"`
// full file path for file logging this repo's zfssync
ZfssyncLogFile string `ini:"-"`
// determines if report_mirror should be run after the sync (default: false)
ReportMirror bool `ini:"report_mirror"`
// add the "-vv" flag to rsync commands and enable the Debug log (default: false)
Verbose bool `ini:"verbose"`
// the repo will write its name and status in a Result struct to DoneChan
@ -232,6 +234,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
RepoLogFile: filepath.Join(newConf.RepoLogDir, repoName) + ".log",
RsyncLogFile: filepath.Join(newConf.RsyncLogDir, repoName) + "-rsync.log",
ZfssyncLogFile: filepath.Join(newConf.ZfssyncLogDir, repoName) + "-zfssync.log",
ReportMirror: false,
Verbose: false,
DoneChan: doneChan,
StopChan: stopChan,

@ -0,0 +1,29 @@
ipv4_address = 129.97.134.71
ipv6_address = 2620:101:f000:4901:c5c::f:1055
; max_jobs = 6
download_dir = /mirror/root
trace_dir = /home/mirror-go/merlin/trace
state_dir = /home/mirror-go/merlin/state
repo_log_dir = /home/mirror-go/merlin/log
rsync_log_dir = /home/mirror-go/merlin/log-rsync
zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
sock_path = /mirror/merlin/run/merlin-go.sock
[ubuntu]
dry_run = true
sync_type = csc-sync-standard
frequency = five-minutely
local_dir = ubuntu
rsync_host = rsync.releases.ubuntu.com
rsync_dir = releases
rsync_exclude = distribution/.timestamp_invisiable, .ignore_me
; add to the repo struct
; RsyncExclude []string `ini:"rsync_exclude"`
; then iterate over RsyncExclude:
; for _, exclude := range repo.RsyncExclude {}

@ -14,7 +14,6 @@ zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
sock_path = /mirror/merlin/run/merlin-go.sock
[ubuntu-releases]
verbose = true
sync_type = csc-sync-standard
frequency = bi-hourly
local_dir = ubuntu-releases
@ -22,7 +21,6 @@ rsync_host = rsync.releases.ubuntu.com
rsync_dir = releases
[xubuntu-releases]
verbose = true
sync_type = csc-sync-standard
frequency = bi-hourly
local_dir = xubuntu-releases
@ -52,6 +50,16 @@ local_dir = CTAN
rsync_host = rsync.dante.ctan.org
rsync_dir = CTAN
[fedora-epel]
dry_run = true
verbose = true
sync_type = csc-sync-standard
frequency = bi-hourly
local_dir = fedora/epel
rsync_host = mirrors.kernel.org
rsync_dir = fedora-epel
report_mirror = true
[cygwin]
sync_type = csc-sync-standard
frequency = twice-daily
@ -133,7 +141,6 @@ rsync_host = download.eclipse.org
rsync_dir = eclipseMirror
[kde]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = kde
@ -141,7 +148,6 @@ rsync_host = rsync.kde.org
rsync_dir = kdeftp
[kde-applicationdata]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = kde-applicationdata
@ -149,7 +155,6 @@ rsync_host = rsync.kde.org
rsync_dir = applicationdata
[artixlinux]
verbose = true
sync_type = csc-sync-standard
frequency = bi-hourly
local_dir = artixlinux
@ -157,7 +162,6 @@ rsync_host = mirror1.artixlinux.org
rsync_dir = artix-linux
[slackware]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = slackware
@ -165,7 +169,6 @@ rsync_host = slackware.cs.utah.edu
rsync_dir = slackware
[x.org]
verbose = true
; csc-sync-standard x.org xorg.freedesktop.org xorg-archive
; csc-sync-standard x.org mirror.us.leaseweb.net xorg
sync_type = csc-sync-standard
@ -175,15 +178,24 @@ rsync_host = rsync.mirrorservice.org
rsync_dir = ftp.x.org/pub/
[centos]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = centos
rsync_host = us-msync.centos.org
rsync_dir = CentOS
[FreeBSD]
[opensuse]
dry_run = true
verbose = true
; unsure what this setting is supposed to do
; #"--exclude distribution/.timestamp_invisible"
sync_type = csc-sync-standard
frequency = bi-hourly
local_dir = opensuse
rsync_host = stage.opensuse.org
rsync_dir = opensuse-full/opensuse/
[FreeBSD]
; Has not updated since at least June 2018
; csc-sync-standard FreeBSD ftp10.us.freebsd.org FreeBSD
sync_type = csc-sync-standard
@ -193,8 +205,18 @@ local_dir = FreeBSD
rsync_host = ftp2.uk.freebsd.org
rsync_dir = ftp.freebsd.org/pub/FreeBSD/
[ubuntu-ports-releases]
[fedora-enchilada]
dry_run = true
verbose = true
; csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null
sync_type = csc-sync-standard
frequency = bi-hourly
local_dir = fedora/linux
rsync_host = mirrors.kernel.org
rsync_dir = fedora-enchilada/linux/
report_mirror = true
[ubuntu-ports-releases]
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = ubuntu-ports-releases
@ -202,7 +224,6 @@ rsync_host = cdimage.ubuntu.com
rsync_dir = cdimage/releases/
[gentoo-portage]
verbose = true
sync_type = csc-sync-standard
frequency = twice-hourly
local_dir = gentoo-portage
@ -226,7 +247,6 @@ rsync_host = mirror.racket-lang.org
rsync_dir = plt-bundles
[xiph]
verbose = true
; csc-sync-standard xiph downloads.xiph.org xiph/releases
sync_type = csc-sync-standard
frequency = twice-daily
@ -244,7 +264,6 @@ rsync_host = rsync.netbsd.org
rsync_dir = NetBSD
[macports-release]
verbose = true
sync_type = csc-sync-standard
frequency = bi-hourly
local_dir = MacPorts/release
@ -252,7 +271,6 @@ rsync_host = rsync.macports.org
rsync_dir = macports/release/
[macports-distfiles]
verbose = true
sync_type = csc-sync-standard
frequency = bi-hourly
local_dir = MacPorts/mpdistfiles
@ -260,7 +278,6 @@ rsync_host = rsync.macports.org
rsync_dir = macports/distfiles/
[sagemath]
verbose = true
; csc-sync-standard sage mirror.clibre.uqam.ca sage
sync_type = csc-sync-standard
frequency = twice-daily
@ -269,7 +286,6 @@ rsync_host = rsync.sagemath.org
rsync_dir = sage
[vlc]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = vlc
@ -277,7 +293,6 @@ rsync_host = rsync.videolan.org
rsync_dir = videolan-ftp
[tdf]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = tdf
@ -285,7 +300,6 @@ rsync_host = rsync.documentfoundation.org
rsync_dir = tdf-pub
[alpine]
verbose = true
sync_type = csc-sync-standard
frequency = hourly
local_dir = alpine
@ -309,7 +323,6 @@ rsync_host = rsync.ipfire.org
rsync_dir = full
[manjaro]
verbose = true
sync_type = csc-sync-standard
frequency = hourly
local_dir = manjaro
@ -317,7 +330,6 @@ rsync_host = mirrorservice.org
rsync_dir = repo.manjaro.org/repos/
[mxlinux]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = mxlinux
@ -325,7 +337,6 @@ rsync_host = mirror.math.princeton.edu
rsync_dir = pub/mxlinux/
[mxlinux-iso]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = mxlinux-iso
@ -333,7 +344,6 @@ rsync_host = mirror.math.princeton.edu
rsync_dir = pub/mxlinux-iso/
[parabola]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = parabola
@ -341,7 +351,6 @@ rsync_host = repo.parabola.nu:875
rsync_dir = repos/
[trisquel-packages]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = trisquel/packages
@ -349,7 +358,6 @@ rsync_host = rsync.trisquel.info
rsync_dir = trisquel.packages/
[trisquel-iso]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = trisquel/iso
@ -357,7 +365,6 @@ rsync_host = rsync.trisquel.info
rsync_dir = trisquel.iso/
[almalinux]
verbose = true
sync_type = csc-sync-standard
frequency = bi-hourly
local_dir = almalinux

@ -3,7 +3,7 @@ Description=Manages synchronization of mirrored projects
After=network.target
[Service]
ExecStart=/home/mirror-go/merlin/merlin /home/mirror-go/merlin/merlin-config.ini
ExecStart=/home/mirror-go/merlin/merlin --config=/home/mirror-go/merlin/merlin-config.ini
WorkingDirectory=/home/mirror-go/merlin
User=mirror
Group=mirror

@ -50,5 +50,6 @@ func SyncCompleted(repo *config.Repo, exit int) {
if exit == config.SUCCESS {
go zfsSync(repo)
go reportMirror(repo)
}
}

@ -146,3 +146,15 @@ func zfsSync(repo *config.Repo) {
}
}
}
func reportMirror(repo *config.Repo) {
if !repo.ReportMirror {
return
}
repo.Logger.Debug("Running report_mirror")
err := exec.Command("/home/mirror/bin/report_mirror").Run()
if err != nil {
repo.Logger.Error(fmt.Errorf("Error running report_mirror: %w", err))
}
}

Loading…
Cancel
Save