minor fixes

merlin/Handoff.md

@@ -29,17 +29,17 @@
 - [ ] archlinux <
 - [x] artixlinx
 - [x] slackware
-- [ ] debian-cd <
+- [x] debian-cd
 - [x] x.org
 - [x] centos
 - [x] opensuse
 - [x] FreeBSD
 - [ ] fedora-enchilada (broken: some attrs not transfered)
 - [x] ubuntu-ports-releases
-- [ ] gentoo-distfiles <
+- [x] gentoo-distfiles
 - [x] gentoo-portage
 - [ ] gutenberg (unstable)
-- [ ] racket-installers <
+- [x] racket-installers
 - [ ] plt-bundles (broken)
 - [x] xiph
 - [ ] netbsd (broken)
@@ -51,7 +51,7 @@
 - [ ] saltstack <
 - [ ] alpine (unstable)
 - [ ] raspbian (broken: too slow and failing)
-- [ ] raspberrypi <
+- [x] raspberrypi
 - [ ] ipfire (unstable)
 - [x] manjaro
 - [x] mxlinux
@@ -60,7 +60,7 @@
 - [x] trisquel-packages
 - [x] trisquel-iso
 - [x] almalinux
-- [ ] ceph <
+- [x] ceph
 
 ### Other Repos
 - debian-cdimage

merlin/README.md

@@ -13,13 +13,13 @@ Then configure `merlin-config.ini` and run using `./merlin`
 
 ### Nice Features To Add
 - detect if an rsync process is stuck (watch the stdout/stderr of the rsync processes)
-- detect if config file is changed and automatically reload (fsnotify)
+- detect if config file is changed and automatically reload ([fsnotify](https://github.com/fsnotify/fsnotify))
 - sort `arthur status` by last time synced
 - respond to `arthur` without blocking main thread
 - state files appear to be getting extra newlines
 - log rotation for log files (also maybe state reporting to prometheus)
 - split off arthur with a more featureful arthur for debugging
-    - last sync runtime, time until next sync
+    - last sync runtime, time until next sync, due to sync (true/false for repo would sync if could)
     - get expected rsync command or repo config
     - details for last 10 syncs (avg time, success rate, data read/written)
 

merlin/merlin-config.ini

@@ -1,12 +1,11 @@
 ipv4_address = 129.97.134.71
 ipv6_address = 2620:101:f000:4901:c5c::f:1055
 
-; max_jobs = 6
+max_jobs = 8
 
 download_dir = /mirror/root
-trace_dir = /home/mirror-go/merlin/trace
-state_dir = /home/mirror-go/merlin/state
 
+state_dir = /home/mirror-go/merlin/state
 repo_log_dir = /home/mirror-go/merlin/log
 rsync_log_dir = /home/mirror-go/merlin/log-rsync
 zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
@@ -108,7 +107,7 @@ local_dir = CRAN
 rsync_user = cran-rsync
 rsync_host = cran.r-project.org
 rsync_dir = /
-password_file = ~/.ssh/id_cran_rsa
+password_file = /home/mirror/.ssh/id_cran_rsa
 
 [CTAN]
 sync_type = csc-sync-standard
@@ -246,7 +245,6 @@ rsync_host = slackware.cs.utah.edu
 rsync_dir = slackware
 
 [debian-cd]
-verbose = true
 sync_type = csc-sync-debian-cd
 frequency = twice-daily
 local_dir = debian-cd
@@ -254,6 +252,7 @@ rsync_host = cdimage.debian.org
 rsync_dir = debian-cd
 
 [x.org]
+; rsync://mirrors.ircam.fr/pub/x.org/
 ; csc-sync-standard x.org xorg.freedesktop.org xorg-archive
 ; csc-sync-standard x.org mirror.us.leaseweb.net xorg
 sync_type = csc-sync-standard
@@ -305,7 +304,6 @@ rsync_host = cdimage.ubuntu.com
 rsync_dir = cdimage/releases/
 
 [gentoo-distfiles]
-verbose = true
 sync_type = csc-sync-standard
 frequency = bi-hourly
 local_dir = gentoo-distfiles
@@ -330,7 +328,6 @@ rsync_host = ftp@ftp.ibiblio.org
 rsync_dir = gutenberg
 
 [racket-installers]
-verbose = true
 ; --cut-dirs=1 is hardcoded for csc-sync-wget
 sync_type = csc-sync-wget
 frequency = twice-daily
@@ -422,7 +419,6 @@ rsync_host = raspbian.freemirror.org
 rsync_dir = raspbian
 
 [raspberrypi]
-verbose = true
 sync_type = csc-sync-standard-ipv6
 frequency = bi-hourly
 local_dir = raspberrypi
@@ -487,7 +483,6 @@ rsync_host = rsync.repo.almalinux.org
 rsync_dir = almalinux/
 
 [ceph]
-verbose = true
 sync_type = csc-sync-ceph
 frequency = tri-hourly
 local_dir = ceph

merlin/merlin.go

@@ -157,6 +157,7 @@ runLoop:
 		}
 
 		if numJobsRunning <= 0 {
+			<-time.After(1 * time.Second) // wait a sec for file writes to complete
 			os.Exit(0)
 		}
 	}

merlin/sync/sync.go

@@ -19,7 +19,7 @@ func runRepoSync(repo *config.Repo) (status int) {
 
 	// run every step of the sync command as long as the previous step was successful
 	for i, args := range cmds {
-		repo.Logger.Debug(fmt.Sprintf("Running step %d of sync", i))
+		repo.Logger.Debug(fmt.Sprintf("Running step %d of sync", i+1))
 
 		status = spawnProcessAndWait(repo, args)
 		if status != config.SUCCESS {
@@ -296,6 +296,7 @@ func cscSyncS3(repo *config.Repo) []string {
 		"nice", "rclone", "sync",
 		"--fast-list",
 		"--use-server-modtime",
+		// "--log-file=" + repo.RepoLogFile,
 		"--bind=" + config.Conf.IPv4Address,
 		"--s3-endpoint=" + repo.RsyncHost, // RCLONE_CONFIG_S3_ENDPOINT=$2
 		"--s3-provider=Other",             // RCLONE_CONFIG_S3_PROVIDER=Other
@@ -313,7 +314,7 @@ func cscSyncSSH(repo *config.Repo) []string {
 		"--stats", "-4",
 	}
 	args = append(args, addConditionalFlags(repo, stdFlags|delete)...)
-	args = append(args, "-e", fmt.Sprintf("\"ssh -b %s -i %s\"", config.Conf.IPv4Address, repo.PasswordFile))
+	args = append(args, "-e", fmt.Sprintf("ssh -b %s -i %s", config.Conf.IPv4Address, repo.PasswordFile))
 	args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo))
 
 	return args
@@ -364,7 +365,7 @@ func cscSyncWget(repo *config.Repo) []string {
 		"--content-disposition",
 		"--execute", "robots=off",
 		"--recursive",
-		"--reject", "\"*\\?*\"",
+		"--reject", "*\\?*",
 		"--directory-prefix=" + buildDownloadDir(repo),
 		repo.RsyncHost,
 	}

merlin/sync/utils.go

@@ -33,7 +33,7 @@ func spawnProcess(repo *config.Repo, args []string) (ch <-chan *exec.Cmd) {
 	// The other env varaibles for rclone s3 were converted to arguments
 	// but I was unable to find a way to convert this one.
 	if repo.SyncType == "csc-sync-s3" {
-		cmd.Env = append(cmd.Env, "RCLONE_CONFIG_S3_TYPE=s3")
+		cmd.Env = append(os.Environ(), "RCLONE_CONFIG_S3_TYPE=s3")
 	}
 
 	repo.Logger.Debug("Starting process")