finalize move and add report_mirror
This commit is contained in:
Andrew Wang
parent
c93e504c58
commit
6025e48049
|
|
@ -6,13 +6,13 @@
|
|||
- [ ] debian-multimedia
|
||||
- [ ] debian-backports
|
||||
- [ ] debian-security
|
||||
- [ ] ubuntu-releases (1 sec sync)
|
||||
- [ ] xubuntu-releases (1 sec sync)
|
||||
- [x] ubuntu-releases
|
||||
- [x] xubuntu-releases
|
||||
- [ ] puppylinux (broken)
|
||||
- [ ] CPAN (unstable)
|
||||
- [ ] CRAN
|
||||
- [x] CTAN
|
||||
- [ ] fedora-epel
|
||||
- [ ] fedora-epel <
|
||||
- [x] cygwin
|
||||
- [x] gnu
|
||||
- [x] nongnu
|
||||
|
|
@ -24,40 +24,40 @@
|
|||
- [x] kernel.org-software
|
||||
- [x] apache
|
||||
- [ ] eclipse (unstable)
|
||||
- [ ] kde <
|
||||
- [ ] kde-appplicationdata <
|
||||
- [x] kde
|
||||
- [x] kde-applicationdata
|
||||
- [ ] archlinux
|
||||
- [ ] artixlinx <
|
||||
- [ ] slackware <
|
||||
- [x] artixlinx
|
||||
- [x] slackware
|
||||
- [ ] debian-cd
|
||||
- [ ] x.org <
|
||||
- [ ] centos <
|
||||
- [ ] opensuse
|
||||
- [ ] FreeBSD <
|
||||
- [ ] fedora-enchilada
|
||||
- [ ] ubuntu-ports-releases <
|
||||
- [x] x.org
|
||||
- [x] centos
|
||||
- [ ] opensuse <
|
||||
- [x] FreeBSD
|
||||
- [ ] fedora-enchilada <
|
||||
- [x] ubuntu-ports-releases
|
||||
- [ ] gentoo-distfiles
|
||||
- [ ] gentoo-portage <
|
||||
- [ ] gutenberg <
|
||||
- [x] gentoo-portage
|
||||
- [ ] gutenberg (broken)
|
||||
- [ ] racket-installers
|
||||
- [ ] plt-bundles <
|
||||
- [ ] xiph <
|
||||
- [ ] netbsd <
|
||||
- [ ] macports-release <
|
||||
- [ ] macports-distfiles <
|
||||
- [ ] sagemath <
|
||||
- [ ] vlc <
|
||||
- [ ] tdf <
|
||||
- [ ] plt-bundles (broken)
|
||||
- [x] xiph
|
||||
- [ ] netbsd (broken)
|
||||
- [x] macports-release
|
||||
- [x] macports-distfiles
|
||||
- [x] sagemath
|
||||
- [x] vlc
|
||||
- [x] tdf
|
||||
- [ ] saltstack
|
||||
- [ ] alpine <
|
||||
- [ ] raspbian <
|
||||
- [x] alpine
|
||||
- [ ] raspbian (high disk usage + long runtime + failing)
|
||||
- [ ] raspberrypi
|
||||
- [ ] ipfire (broken)
|
||||
- [ ] manjaro <
|
||||
- [ ] mxlinux <
|
||||
- [ ] mxlinux-iso <
|
||||
- [ ] parabola <
|
||||
- [ ] trisquel-packages <
|
||||
- [ ] trisquel-iso <
|
||||
- [ ] almalinux <
|
||||
- [x] manjaro
|
||||
- [x] mxlinux
|
||||
- [x] mxlinux-iso
|
||||
- [x] parabola
|
||||
- [x] trisquel-packages
|
||||
- [x] trisquel-iso
|
||||
- [x] almalinux
|
||||
- [ ] ceph
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ Then configure `merlin-config.ini` and run using `./merlin`
|
|||
- get the config or the rsync commond that a repo will sync with using a cli tool
|
||||
- improve conversion from exit status enum to string
|
||||
- sort `arthur status` by last time synced
|
||||
- time spent to sync during last sync, seconds/minutes/hours until next sync
|
||||
- last sync runtime, time until next sync
|
||||
- when sync fails, then make a copy of the rsync logs
|
||||
- create `csc-sync-fedora` rather than using the `report_mirror` setting
|
||||
|
||||
### Completed
|
||||
- [x] add bwlimit option for each rsync process
|
||||
|
|
|
|||
|
|
@ -132,6 +132,8 @@ type Repo struct {
|
|||
RsyncLogFile string `ini:"-"`
|
||||
// full file path for file logging this repo's zfssync
|
||||
ZfssyncLogFile string `ini:"-"`
|
||||
// determines if report_mirror should be run after the sync (default: false)
|
||||
ReportMirror bool `ini:"report_mirror"`
|
||||
// add the "-vv" flag to rsync commands and enable the Debug log (default: false)
|
||||
Verbose bool `ini:"verbose"`
|
||||
// the repo will write its name and status in a Result struct to DoneChan
|
||||
|
|
@ -232,6 +234,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
|
|||
RepoLogFile: filepath.Join(newConf.RepoLogDir, repoName) + ".log",
|
||||
RsyncLogFile: filepath.Join(newConf.RsyncLogDir, repoName) + "-rsync.log",
|
||||
ZfssyncLogFile: filepath.Join(newConf.ZfssyncLogDir, repoName) + "-zfssync.log",
|
||||
ReportMirror: false,
|
||||
Verbose: false,
|
||||
DoneChan: doneChan,
|
||||
StopChan: stopChan,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
ipv4_address = 129.97.134.71
|
||||
ipv6_address = 2620:101:f000:4901:c5c::f:1055
|
||||
|
||||
; max_jobs = 6
|
||||
|
||||
download_dir = /mirror/root
|
||||
trace_dir = /home/mirror-go/merlin/trace
|
||||
state_dir = /home/mirror-go/merlin/state
|
||||
|
||||
repo_log_dir = /home/mirror-go/merlin/log
|
||||
rsync_log_dir = /home/mirror-go/merlin/log-rsync
|
||||
zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
|
||||
|
||||
sock_path = /mirror/merlin/run/merlin-go.sock
|
||||
|
||||
[ubuntu]
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = five-minutely
|
||||
local_dir = ubuntu
|
||||
rsync_host = rsync.releases.ubuntu.com
|
||||
rsync_dir = releases
|
||||
rsync_exclude = distribution/.timestamp_invisiable, .ignore_me
|
||||
|
||||
; add to the repo struct
|
||||
; RsyncExclude []string `ini:"rsync_exclude"`
|
||||
|
||||
; then iterate over RsyncExclude:
|
||||
; for _, exclude := range repo.RsyncExclude {}
|
||||
|
|
@ -14,7 +14,6 @@ zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
|
|||
sock_path = /mirror/merlin/run/merlin-go.sock
|
||||
|
||||
[ubuntu-releases]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
local_dir = ubuntu-releases
|
||||
|
|
@ -22,7 +21,6 @@ rsync_host = rsync.releases.ubuntu.com
|
|||
rsync_dir = releases
|
||||
|
||||
[xubuntu-releases]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
local_dir = xubuntu-releases
|
||||
|
|
@ -52,6 +50,16 @@ local_dir = CTAN
|
|||
rsync_host = rsync.dante.ctan.org
|
||||
rsync_dir = CTAN
|
||||
|
||||
[fedora-epel]
|
||||
dry_run = true
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
local_dir = fedora/epel
|
||||
rsync_host = mirrors.kernel.org
|
||||
rsync_dir = fedora-epel
|
||||
report_mirror = true
|
||||
|
||||
[cygwin]
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
|
|
@ -133,7 +141,6 @@ rsync_host = download.eclipse.org
|
|||
rsync_dir = eclipseMirror
|
||||
|
||||
[kde]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = kde
|
||||
|
|
@ -141,7 +148,6 @@ rsync_host = rsync.kde.org
|
|||
rsync_dir = kdeftp
|
||||
|
||||
[kde-applicationdata]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = kde-applicationdata
|
||||
|
|
@ -149,7 +155,6 @@ rsync_host = rsync.kde.org
|
|||
rsync_dir = applicationdata
|
||||
|
||||
[artixlinux]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
local_dir = artixlinux
|
||||
|
|
@ -157,7 +162,6 @@ rsync_host = mirror1.artixlinux.org
|
|||
rsync_dir = artix-linux
|
||||
|
||||
[slackware]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = slackware
|
||||
|
|
@ -165,7 +169,6 @@ rsync_host = slackware.cs.utah.edu
|
|||
rsync_dir = slackware
|
||||
|
||||
[x.org]
|
||||
verbose = true
|
||||
; csc-sync-standard x.org xorg.freedesktop.org xorg-archive
|
||||
; csc-sync-standard x.org mirror.us.leaseweb.net xorg
|
||||
sync_type = csc-sync-standard
|
||||
|
|
@ -175,15 +178,24 @@ rsync_host = rsync.mirrorservice.org
|
|||
rsync_dir = ftp.x.org/pub/
|
||||
|
||||
[centos]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = centos
|
||||
rsync_host = us-msync.centos.org
|
||||
rsync_dir = CentOS
|
||||
|
||||
[FreeBSD]
|
||||
[opensuse]
|
||||
dry_run = true
|
||||
verbose = true
|
||||
; unsure what this setting is supposed to do
|
||||
; #"--exclude distribution/.timestamp_invisible"
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
local_dir = opensuse
|
||||
rsync_host = stage.opensuse.org
|
||||
rsync_dir = opensuse-full/opensuse/
|
||||
|
||||
[FreeBSD]
|
||||
; Has not updated since at least June 2018
|
||||
; csc-sync-standard FreeBSD ftp10.us.freebsd.org FreeBSD
|
||||
sync_type = csc-sync-standard
|
||||
|
|
@ -193,8 +205,18 @@ local_dir = FreeBSD
|
|||
rsync_host = ftp2.uk.freebsd.org
|
||||
rsync_dir = ftp.freebsd.org/pub/FreeBSD/
|
||||
|
||||
[ubuntu-ports-releases]
|
||||
[fedora-enchilada]
|
||||
dry_run = true
|
||||
verbose = true
|
||||
; csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
local_dir = fedora/linux
|
||||
rsync_host = mirrors.kernel.org
|
||||
rsync_dir = fedora-enchilada/linux/
|
||||
report_mirror = true
|
||||
|
||||
[ubuntu-ports-releases]
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = ubuntu-ports-releases
|
||||
|
|
@ -202,7 +224,6 @@ rsync_host = cdimage.ubuntu.com
|
|||
rsync_dir = cdimage/releases/
|
||||
|
||||
[gentoo-portage]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-hourly
|
||||
local_dir = gentoo-portage
|
||||
|
|
@ -226,7 +247,6 @@ rsync_host = mirror.racket-lang.org
|
|||
rsync_dir = plt-bundles
|
||||
|
||||
[xiph]
|
||||
verbose = true
|
||||
; csc-sync-standard xiph downloads.xiph.org xiph/releases
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
|
|
@ -244,7 +264,6 @@ rsync_host = rsync.netbsd.org
|
|||
rsync_dir = NetBSD
|
||||
|
||||
[macports-release]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
local_dir = MacPorts/release
|
||||
|
|
@ -252,7 +271,6 @@ rsync_host = rsync.macports.org
|
|||
rsync_dir = macports/release/
|
||||
|
||||
[macports-distfiles]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
local_dir = MacPorts/mpdistfiles
|
||||
|
|
@ -260,7 +278,6 @@ rsync_host = rsync.macports.org
|
|||
rsync_dir = macports/distfiles/
|
||||
|
||||
[sagemath]
|
||||
verbose = true
|
||||
; csc-sync-standard sage mirror.clibre.uqam.ca sage
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
|
|
@ -269,7 +286,6 @@ rsync_host = rsync.sagemath.org
|
|||
rsync_dir = sage
|
||||
|
||||
[vlc]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = vlc
|
||||
|
|
@ -277,7 +293,6 @@ rsync_host = rsync.videolan.org
|
|||
rsync_dir = videolan-ftp
|
||||
|
||||
[tdf]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = tdf
|
||||
|
|
@ -285,7 +300,6 @@ rsync_host = rsync.documentfoundation.org
|
|||
rsync_dir = tdf-pub
|
||||
|
||||
[alpine]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = hourly
|
||||
local_dir = alpine
|
||||
|
|
@ -309,7 +323,6 @@ rsync_host = rsync.ipfire.org
|
|||
rsync_dir = full
|
||||
|
||||
[manjaro]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = hourly
|
||||
local_dir = manjaro
|
||||
|
|
@ -317,7 +330,6 @@ rsync_host = mirrorservice.org
|
|||
rsync_dir = repo.manjaro.org/repos/
|
||||
|
||||
[mxlinux]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = mxlinux
|
||||
|
|
@ -325,7 +337,6 @@ rsync_host = mirror.math.princeton.edu
|
|||
rsync_dir = pub/mxlinux/
|
||||
|
||||
[mxlinux-iso]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = mxlinux-iso
|
||||
|
|
@ -333,7 +344,6 @@ rsync_host = mirror.math.princeton.edu
|
|||
rsync_dir = pub/mxlinux-iso/
|
||||
|
||||
[parabola]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = parabola
|
||||
|
|
@ -341,7 +351,6 @@ rsync_host = repo.parabola.nu:875
|
|||
rsync_dir = repos/
|
||||
|
||||
[trisquel-packages]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = trisquel/packages
|
||||
|
|
@ -349,7 +358,6 @@ rsync_host = rsync.trisquel.info
|
|||
rsync_dir = trisquel.packages/
|
||||
|
||||
[trisquel-iso]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = trisquel/iso
|
||||
|
|
@ -357,7 +365,6 @@ rsync_host = rsync.trisquel.info
|
|||
rsync_dir = trisquel.iso/
|
||||
|
||||
[almalinux]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
local_dir = almalinux
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ Description=Manages synchronization of mirrored projects
|
|||
After=network.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/home/mirror-go/merlin/merlin /home/mirror-go/merlin/merlin-config.ini
|
||||
ExecStart=/home/mirror-go/merlin/merlin --config=/home/mirror-go/merlin/merlin-config.ini
|
||||
WorkingDirectory=/home/mirror-go/merlin
|
||||
User=mirror
|
||||
Group=mirror
|
||||
|
|
|
|||
|
|
@ -50,5 +50,6 @@ func SyncCompleted(repo *config.Repo, exit int) {
|
|||
|
||||
if exit == config.SUCCESS {
|
||||
go zfsSync(repo)
|
||||
go reportMirror(repo)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -146,3 +146,15 @@ func zfsSync(repo *config.Repo) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func reportMirror(repo *config.Repo) {
|
||||
if !repo.ReportMirror {
|
||||
return
|
||||
}
|
||||
|
||||
repo.Logger.Debug("Running report_mirror")
|
||||
err := exec.Command("/home/mirror/bin/report_mirror").Run()
|
||||
if err != nil {
|
||||
repo.Logger.Error(fmt.Errorf("Error running report_mirror: %w", err))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue