diff --git a/merlin/Handoff.md b/merlin/Handoff.md index dd767d1..ecbdf55 100644 --- a/merlin/Handoff.md +++ b/merlin/Handoff.md @@ -29,17 +29,17 @@ - [ ] archlinux < - [x] artixlinx - [x] slackware -- [ ] debian-cd < +- [x] debian-cd - [x] x.org - [x] centos - [x] opensuse - [x] FreeBSD - [ ] fedora-enchilada (broken: some attrs not transfered) - [x] ubuntu-ports-releases -- [ ] gentoo-distfiles < +- [x] gentoo-distfiles - [x] gentoo-portage - [ ] gutenberg (unstable) -- [ ] racket-installers < +- [x] racket-installers - [ ] plt-bundles (broken) - [x] xiph - [ ] netbsd (broken) @@ -51,7 +51,7 @@ - [ ] saltstack < - [ ] alpine (unstable) - [ ] raspbian (broken: too slow and failing) -- [ ] raspberrypi < +- [x] raspberrypi - [ ] ipfire (unstable) - [x] manjaro - [x] mxlinux @@ -60,7 +60,7 @@ - [x] trisquel-packages - [x] trisquel-iso - [x] almalinux -- [ ] ceph < +- [x] ceph ### Other Repos - debian-cdimage diff --git a/merlin/README.md b/merlin/README.md index 403d46a..8995c00 100644 --- a/merlin/README.md +++ b/merlin/README.md @@ -13,13 +13,13 @@ Then configure `merlin-config.ini` and run using `./merlin` ### Nice Features To Add - detect if an rsync process is stuck (watch the stdout/stderr of the rsync processes) -- detect if config file is changed and automatically reload (fsnotify) +- detect if config file is changed and automatically reload ([fsnotify](https://github.com/fsnotify/fsnotify)) - sort `arthur status` by last time synced - respond to `arthur` without blocking main thread - state files appear to be getting extra newlines - log rotation for log files (also maybe state reporting to prometheus) - split off arthur with a more featureful arthur for debugging - - last sync runtime, time until next sync + - last sync runtime, time until next sync, due to sync (true/false for repo would sync if could) - get expected rsync command or repo config - details for last 10 syncs (avg time, success rate, data read/written) diff --git a/merlin/merlin-config.ini b/merlin/merlin-config.ini index e7fc695..afc9eb5 100644 --- a/merlin/merlin-config.ini +++ b/merlin/merlin-config.ini @@ -1,12 +1,11 @@ ipv4_address = 129.97.134.71 ipv6_address = 2620:101:f000:4901:c5c::f:1055 -; max_jobs = 6 +max_jobs = 8 download_dir = /mirror/root -trace_dir = /home/mirror-go/merlin/trace -state_dir = /home/mirror-go/merlin/state +state_dir = /home/mirror-go/merlin/state repo_log_dir = /home/mirror-go/merlin/log rsync_log_dir = /home/mirror-go/merlin/log-rsync zfssync_log_dir = /home/mirror-go/merlin/log-zfssync @@ -108,7 +107,7 @@ local_dir = CRAN rsync_user = cran-rsync rsync_host = cran.r-project.org rsync_dir = / -password_file = ~/.ssh/id_cran_rsa +password_file = /home/mirror/.ssh/id_cran_rsa [CTAN] sync_type = csc-sync-standard @@ -246,7 +245,6 @@ rsync_host = slackware.cs.utah.edu rsync_dir = slackware [debian-cd] -verbose = true sync_type = csc-sync-debian-cd frequency = twice-daily local_dir = debian-cd @@ -254,6 +252,7 @@ rsync_host = cdimage.debian.org rsync_dir = debian-cd [x.org] +; rsync://mirrors.ircam.fr/pub/x.org/ ; csc-sync-standard x.org xorg.freedesktop.org xorg-archive ; csc-sync-standard x.org mirror.us.leaseweb.net xorg sync_type = csc-sync-standard @@ -305,7 +304,6 @@ rsync_host = cdimage.ubuntu.com rsync_dir = cdimage/releases/ [gentoo-distfiles] -verbose = true sync_type = csc-sync-standard frequency = bi-hourly local_dir = gentoo-distfiles @@ -330,7 +328,6 @@ rsync_host = ftp@ftp.ibiblio.org rsync_dir = gutenberg [racket-installers] -verbose = true ; --cut-dirs=1 is hardcoded for csc-sync-wget sync_type = csc-sync-wget frequency = twice-daily @@ -422,7 +419,6 @@ rsync_host = raspbian.freemirror.org rsync_dir = raspbian [raspberrypi] -verbose = true sync_type = csc-sync-standard-ipv6 frequency = bi-hourly local_dir = raspberrypi @@ -487,7 +483,6 @@ rsync_host = rsync.repo.almalinux.org rsync_dir = almalinux/ [ceph] -verbose = true sync_type = csc-sync-ceph frequency = tri-hourly local_dir = ceph diff --git a/merlin/merlin.go b/merlin/merlin.go index 17f49c6..4122402 100644 --- a/merlin/merlin.go +++ b/merlin/merlin.go @@ -157,6 +157,7 @@ runLoop: } if numJobsRunning <= 0 { + <-time.After(1 * time.Second) // wait a sec for file writes to complete os.Exit(0) } } diff --git a/merlin/sync/sync.go b/merlin/sync/sync.go index 4ead8f4..2cb8eee 100644 --- a/merlin/sync/sync.go +++ b/merlin/sync/sync.go @@ -19,7 +19,7 @@ func runRepoSync(repo *config.Repo) (status int) { // run every step of the sync command as long as the previous step was successful for i, args := range cmds { - repo.Logger.Debug(fmt.Sprintf("Running step %d of sync", i)) + repo.Logger.Debug(fmt.Sprintf("Running step %d of sync", i+1)) status = spawnProcessAndWait(repo, args) if status != config.SUCCESS { @@ -296,6 +296,7 @@ func cscSyncS3(repo *config.Repo) []string { "nice", "rclone", "sync", "--fast-list", "--use-server-modtime", + // "--log-file=" + repo.RepoLogFile, "--bind=" + config.Conf.IPv4Address, "--s3-endpoint=" + repo.RsyncHost, // RCLONE_CONFIG_S3_ENDPOINT=$2 "--s3-provider=Other", // RCLONE_CONFIG_S3_PROVIDER=Other @@ -313,7 +314,7 @@ func cscSyncSSH(repo *config.Repo) []string { "--stats", "-4", } args = append(args, addConditionalFlags(repo, stdFlags|delete)...) - args = append(args, "-e", fmt.Sprintf("\"ssh -b %s -i %s\"", config.Conf.IPv4Address, repo.PasswordFile)) + args = append(args, "-e", fmt.Sprintf("ssh -b %s -i %s", config.Conf.IPv4Address, repo.PasswordFile)) args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo)) return args @@ -364,7 +365,7 @@ func cscSyncWget(repo *config.Repo) []string { "--content-disposition", "--execute", "robots=off", "--recursive", - "--reject", "\"*\\?*\"", + "--reject", "*\\?*", "--directory-prefix=" + buildDownloadDir(repo), repo.RsyncHost, } diff --git a/merlin/sync/utils.go b/merlin/sync/utils.go index 6f90f35..4f05c00 100644 --- a/merlin/sync/utils.go +++ b/merlin/sync/utils.go @@ -33,7 +33,7 @@ func spawnProcess(repo *config.Repo, args []string) (ch <-chan *exec.Cmd) { // The other env varaibles for rclone s3 were converted to arguments // but I was unable to find a way to convert this one. if repo.SyncType == "csc-sync-s3" { - cmd.Env = append(cmd.Env, "RCLONE_CONFIG_S3_TYPE=s3") + cmd.Env = append(os.Environ(), "RCLONE_CONFIG_S3_TYPE=s3") } repo.Logger.Debug("Starting process")