minor fixes
This commit is contained in:
parent
1d852fa8cc
commit
3c7cb13769
|
@ -29,17 +29,17 @@
|
|||
- [ ] archlinux <
|
||||
- [x] artixlinx
|
||||
- [x] slackware
|
||||
- [ ] debian-cd <
|
||||
- [x] debian-cd
|
||||
- [x] x.org
|
||||
- [x] centos
|
||||
- [x] opensuse
|
||||
- [x] FreeBSD
|
||||
- [ ] fedora-enchilada (broken: some attrs not transfered)
|
||||
- [x] ubuntu-ports-releases
|
||||
- [ ] gentoo-distfiles <
|
||||
- [x] gentoo-distfiles
|
||||
- [x] gentoo-portage
|
||||
- [ ] gutenberg (unstable)
|
||||
- [ ] racket-installers <
|
||||
- [x] racket-installers
|
||||
- [ ] plt-bundles (broken)
|
||||
- [x] xiph
|
||||
- [ ] netbsd (broken)
|
||||
|
@ -51,7 +51,7 @@
|
|||
- [ ] saltstack <
|
||||
- [ ] alpine (unstable)
|
||||
- [ ] raspbian (broken: too slow and failing)
|
||||
- [ ] raspberrypi <
|
||||
- [x] raspberrypi
|
||||
- [ ] ipfire (unstable)
|
||||
- [x] manjaro
|
||||
- [x] mxlinux
|
||||
|
@ -60,7 +60,7 @@
|
|||
- [x] trisquel-packages
|
||||
- [x] trisquel-iso
|
||||
- [x] almalinux
|
||||
- [ ] ceph <
|
||||
- [x] ceph
|
||||
|
||||
### Other Repos
|
||||
- debian-cdimage
|
||||
|
|
|
@ -13,13 +13,13 @@ Then configure `merlin-config.ini` and run using `./merlin`
|
|||
|
||||
### Nice Features To Add
|
||||
- detect if an rsync process is stuck (watch the stdout/stderr of the rsync processes)
|
||||
- detect if config file is changed and automatically reload (fsnotify)
|
||||
- detect if config file is changed and automatically reload ([fsnotify](https://github.com/fsnotify/fsnotify))
|
||||
- sort `arthur status` by last time synced
|
||||
- respond to `arthur` without blocking main thread
|
||||
- state files appear to be getting extra newlines
|
||||
- log rotation for log files (also maybe state reporting to prometheus)
|
||||
- split off arthur with a more featureful arthur for debugging
|
||||
- last sync runtime, time until next sync
|
||||
- last sync runtime, time until next sync, due to sync (true/false for repo would sync if could)
|
||||
- get expected rsync command or repo config
|
||||
- details for last 10 syncs (avg time, success rate, data read/written)
|
||||
|
||||
|
|
|
@ -1,12 +1,11 @@
|
|||
ipv4_address = 129.97.134.71
|
||||
ipv6_address = 2620:101:f000:4901:c5c::f:1055
|
||||
|
||||
; max_jobs = 6
|
||||
max_jobs = 8
|
||||
|
||||
download_dir = /mirror/root
|
||||
trace_dir = /home/mirror-go/merlin/trace
|
||||
state_dir = /home/mirror-go/merlin/state
|
||||
|
||||
state_dir = /home/mirror-go/merlin/state
|
||||
repo_log_dir = /home/mirror-go/merlin/log
|
||||
rsync_log_dir = /home/mirror-go/merlin/log-rsync
|
||||
zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
|
||||
|
@ -108,7 +107,7 @@ local_dir = CRAN
|
|||
rsync_user = cran-rsync
|
||||
rsync_host = cran.r-project.org
|
||||
rsync_dir = /
|
||||
password_file = ~/.ssh/id_cran_rsa
|
||||
password_file = /home/mirror/.ssh/id_cran_rsa
|
||||
|
||||
[CTAN]
|
||||
sync_type = csc-sync-standard
|
||||
|
@ -246,7 +245,6 @@ rsync_host = slackware.cs.utah.edu
|
|||
rsync_dir = slackware
|
||||
|
||||
[debian-cd]
|
||||
verbose = true
|
||||
sync_type = csc-sync-debian-cd
|
||||
frequency = twice-daily
|
||||
local_dir = debian-cd
|
||||
|
@ -254,6 +252,7 @@ rsync_host = cdimage.debian.org
|
|||
rsync_dir = debian-cd
|
||||
|
||||
[x.org]
|
||||
; rsync://mirrors.ircam.fr/pub/x.org/
|
||||
; csc-sync-standard x.org xorg.freedesktop.org xorg-archive
|
||||
; csc-sync-standard x.org mirror.us.leaseweb.net xorg
|
||||
sync_type = csc-sync-standard
|
||||
|
@ -305,7 +304,6 @@ rsync_host = cdimage.ubuntu.com
|
|||
rsync_dir = cdimage/releases/
|
||||
|
||||
[gentoo-distfiles]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
local_dir = gentoo-distfiles
|
||||
|
@ -330,7 +328,6 @@ rsync_host = ftp@ftp.ibiblio.org
|
|||
rsync_dir = gutenberg
|
||||
|
||||
[racket-installers]
|
||||
verbose = true
|
||||
; --cut-dirs=1 is hardcoded for csc-sync-wget
|
||||
sync_type = csc-sync-wget
|
||||
frequency = twice-daily
|
||||
|
@ -422,7 +419,6 @@ rsync_host = raspbian.freemirror.org
|
|||
rsync_dir = raspbian
|
||||
|
||||
[raspberrypi]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard-ipv6
|
||||
frequency = bi-hourly
|
||||
local_dir = raspberrypi
|
||||
|
@ -487,7 +483,6 @@ rsync_host = rsync.repo.almalinux.org
|
|||
rsync_dir = almalinux/
|
||||
|
||||
[ceph]
|
||||
verbose = true
|
||||
sync_type = csc-sync-ceph
|
||||
frequency = tri-hourly
|
||||
local_dir = ceph
|
||||
|
|
|
@ -157,6 +157,7 @@ runLoop:
|
|||
}
|
||||
|
||||
if numJobsRunning <= 0 {
|
||||
<-time.After(1 * time.Second) // wait a sec for file writes to complete
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,7 +19,7 @@ func runRepoSync(repo *config.Repo) (status int) {
|
|||
|
||||
// run every step of the sync command as long as the previous step was successful
|
||||
for i, args := range cmds {
|
||||
repo.Logger.Debug(fmt.Sprintf("Running step %d of sync", i))
|
||||
repo.Logger.Debug(fmt.Sprintf("Running step %d of sync", i+1))
|
||||
|
||||
status = spawnProcessAndWait(repo, args)
|
||||
if status != config.SUCCESS {
|
||||
|
@ -296,6 +296,7 @@ func cscSyncS3(repo *config.Repo) []string {
|
|||
"nice", "rclone", "sync",
|
||||
"--fast-list",
|
||||
"--use-server-modtime",
|
||||
// "--log-file=" + repo.RepoLogFile,
|
||||
"--bind=" + config.Conf.IPv4Address,
|
||||
"--s3-endpoint=" + repo.RsyncHost, // RCLONE_CONFIG_S3_ENDPOINT=$2
|
||||
"--s3-provider=Other", // RCLONE_CONFIG_S3_PROVIDER=Other
|
||||
|
@ -313,7 +314,7 @@ func cscSyncSSH(repo *config.Repo) []string {
|
|||
"--stats", "-4",
|
||||
}
|
||||
args = append(args, addConditionalFlags(repo, stdFlags|delete)...)
|
||||
args = append(args, "-e", fmt.Sprintf("\"ssh -b %s -i %s\"", config.Conf.IPv4Address, repo.PasswordFile))
|
||||
args = append(args, "-e", fmt.Sprintf("ssh -b %s -i %s", config.Conf.IPv4Address, repo.PasswordFile))
|
||||
args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo))
|
||||
|
||||
return args
|
||||
|
@ -364,7 +365,7 @@ func cscSyncWget(repo *config.Repo) []string {
|
|||
"--content-disposition",
|
||||
"--execute", "robots=off",
|
||||
"--recursive",
|
||||
"--reject", "\"*\\?*\"",
|
||||
"--reject", "*\\?*",
|
||||
"--directory-prefix=" + buildDownloadDir(repo),
|
||||
repo.RsyncHost,
|
||||
}
|
||||
|
|
|
@ -33,7 +33,7 @@ func spawnProcess(repo *config.Repo, args []string) (ch <-chan *exec.Cmd) {
|
|||
// The other env varaibles for rclone s3 were converted to arguments
|
||||
// but I was unable to find a way to convert this one.
|
||||
if repo.SyncType == "csc-sync-s3" {
|
||||
cmd.Env = append(cmd.Env, "RCLONE_CONFIG_S3_TYPE=s3")
|
||||
cmd.Env = append(os.Environ(), "RCLONE_CONFIG_S3_TYPE=s3")
|
||||
}
|
||||
|
||||
repo.Logger.Debug("Starting process")
|
||||
|
|
Loading…
Reference in New Issue