finish adding all projects

merlin/Handoff.md

@@ -1,7 +1,7 @@
 ## Status of moving from merlin to merlin-go
 - [x] debian
-- [ ] ubuntu <
+- [ ] ubuntu (successful but too slow)
-- [ ] ubuntu-ports <
+- [ ] ubuntu-ports (successful but too slow)
 - [x] linuxmint-packages
 - [x] debian-multimedia
 - [ ] debian-backports (part 1 broken)
@@ -10,7 +10,7 @@
 - [x] xubuntu-releases
 - [ ] puppylinux (broken)
 - [ ] CPAN (unstable)
-<!-- - [ ] CRAN -->
+- [ ] CRAN <
 - [x] CTAN
 - [ ] fedora-epel (some attrs not transfered)
 - [x] cygwin
@@ -26,20 +26,20 @@
 - [ ] eclipse (unstable)
 - [x] kde
 - [x] kde-applicationdata
-<!-- - [ ] archlinux -->
+- [ ] archlinux <
 - [x] artixlinx
 - [x] slackware
-<!-- - [ ] debian-cd -->
+- [ ] debian-cd <
 - [x] x.org
 - [x] centos
 - [x] opensuse
 - [x] FreeBSD
 - [ ] fedora-enchilada (broken: some attrs not transfered)
 - [x] ubuntu-ports-releases
-<!-- - [ ] gentoo-distfiles -->
+- [ ] gentoo-distfiles <
 - [x] gentoo-portage
 - [ ] gutenberg (unstable)
-<!-- - [ ] racket-installers -->
+- [ ] racket-installers <
 - [ ] plt-bundles (broken)
 - [x] xiph
 - [ ] netbsd (broken)
@@ -48,9 +48,10 @@
 - [x] sagemath
 - [x] vlc
 - [x] tdf
-<!-- - [ ] saltstack -->
+- [ ] saltstack <
 - [ ] alpine (unstable)
-- [ ] raspbian (broekn: high disk usage + long runtime + failing)
+- [ ] raspbian (broken: too slow and failing)
+- [ ] raspberrypi <
 - [ ] ipfire (unstable)
 - [x] manjaro
 - [x] mxlinux
@@ -59,7 +60,7 @@
 - [x] trisquel-packages
 - [x] trisquel-iso
 - [x] almalinux
-<!-- - [ ] ceph -->
+- [ ] ceph <
 
 ### Other Repos
 - debian-cdimage

merlin/README.md

@@ -16,6 +16,8 @@ Then configure `merlin-config.ini` and run using `./merlin`
 - detect if config file is changed and automatically reload (fsnotify)
 - sort `arthur status` by last time synced
 - respond to `arthur` without blocking main thread
+- state files appear to be getting extra newlines
+- log rotation for log files (also maybe state reporting to prometheus)
 - split off arthur with a more featureful arthur for debugging
     - last sync runtime, time until next sync
     - get expected rsync command or repo config

merlin/config/config.go

@@ -61,9 +61,9 @@ func StatusToString(status int) string {
 	case TERMINATED:
 		return "terminated"
 	case SUCCESS:
-		return "success"
+		return "completed"
 	case FAILURE:
-		return "failure"
+		return "failed"
 	case NOT_RUN_YET:
 		return "not run yet"
 	}

merlin/merlin-config.ini

@@ -13,6 +13,8 @@ zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
 
 sock_path = /mirror/merlin/run/merlin-go.sock
 
+; note: the .ssh keys are only located on /home/mirror/.ssh not at /home/mirror-go/.ssh
+
 [debian]
 sync_type = csc-sync-debian
 frequency = bi-hourly
@@ -21,6 +23,7 @@ rsync_host = debian.mirror.rafal.ca
 rsync_dir = debian
 
 [ubuntu]
+verbose = true
 sync_type = csc-sync-debian
 frequency = bi-hourly
 local_dir = ubuntu
@@ -29,6 +32,7 @@ rsync_dir = ubuntu
 trace_host = drescher.canonical.com
 
 [ubuntu-ports]
+verbose = true
 sync_type = csc-sync-debian
 frequency = bi-hourly
 local_dir = ubuntu-ports
@@ -51,6 +55,7 @@ rsync_host = www.deb-multimedia.org
 rsync_dir = deb
 
 [debian-backports]
+verbose = true
 sync_type = csc-sync-debian
 frequency = bi-hourly
 local_dir = debian-backports
@@ -95,6 +100,16 @@ local_dir = CPAN
 rsync_host = cpan-rsync.perl.org
 rsync_dir = CPAN
 
+[CRAN]
+verbose = true
+sync_type = csc-sync-ssh
+frequency = twice-daily
+local_dir = CRAN
+rsync_user = cran-rsync
+rsync_host = cran.r-project.org
+rsync_dir = /
+password_file = ~/.ssh/id_cran_rsa
+
 [CTAN]
 sync_type = csc-sync-standard
 frequency = twice-daily
@@ -204,6 +219,18 @@ local_dir = kde-applicationdata
 rsync_host = rsync.kde.org
 rsync_dir = applicationdata
 
+; We are a Tier 1 arch mirror (https://bugs.archlinux.org/task/52853) so our IP is important.
+; our rsync_host is rsync://rsync.archlinux.org/ftp_tier1
+[archlinux]
+; csc-sync-standard archlinux archlinux.mirror.rafal.ca archlinux
+verbose = true
+sync_type = csc-sync-archlinux
+frequency = five-minutely
+local_dir = archlinux
+rsync_host = rsync.archlinux.org
+rsync_dir = ftp_tier1
+trace_host = http://rsync.archlinux.org/lastupdate
+
 [artixlinux]
 sync_type = csc-sync-standard
 frequency = bi-hourly
@@ -218,6 +245,14 @@ local_dir = slackware
 rsync_host = slackware.cs.utah.edu
 rsync_dir = slackware
 
+[debian-cd]
+verbose = true
+sync_type = csc-sync-debian-cd
+frequency = twice-daily
+local_dir = debian-cd
+rsync_host = cdimage.debian.org
+rsync_dir = debian-cd
+
 [x.org]
 ; csc-sync-standard x.org xorg.freedesktop.org xorg-archive
 ; csc-sync-standard x.org mirror.us.leaseweb.net xorg
@@ -269,6 +304,16 @@ local_dir = ubuntu-ports-releases
 rsync_host = cdimage.ubuntu.com
 rsync_dir = cdimage/releases/
 
+[gentoo-distfiles]
+verbose = true
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = gentoo-distfiles
+rsync_user = gentoo
+rsync_host = masterdistfiles.gentoo.org
+rsync_dir = gentoo
+password_file = /home/mirror-go/passwords/gentoo-distfiles
+
 [gentoo-portage]
 sync_type = csc-sync-standard
 frequency = twice-hourly
@@ -284,6 +329,14 @@ local_dir = gutenberg
 rsync_host = ftp@ftp.ibiblio.org
 rsync_dir = gutenberg
 
+[racket-installers]
+verbose = true
+; --cut-dirs=1 is hardcoded for csc-sync-wget
+sync_type = csc-sync-wget
+frequency = twice-daily
+local_dir = racket/racket-installers
+rsync_host = https://mirror.racket-lang.org/installers/
+
 [plt-bundles]
 verbose = true
 sync_type = csc-sync-standard
@@ -345,6 +398,13 @@ local_dir = tdf
 rsync_host = rsync.documentfoundation.org
 rsync_dir = tdf-pub
 
+[saltstack]
+verbose = true
+sync_type = csc-sync-s3
+frequency = daily
+local_dir = saltstack
+rsync_host = https://s3.repo.saltproject.io
+
 [alpine]
 verbose = true
 sync_type = csc-sync-standard
@@ -361,6 +421,14 @@ local_dir = raspbian
 rsync_host = raspbian.freemirror.org
 rsync_dir = raspbian
 
+[raspberrypi]
+verbose = true
+sync_type = csc-sync-standard-ipv6
+frequency = bi-hourly
+local_dir = raspberrypi
+rsync_host = apt-repo.raspberrypi.org
+rsync_dir = archive
+
 [ipfire]
 verbose = true
 sync_type = csc-sync-standard
@@ -417,3 +485,10 @@ frequency = bi-hourly
 local_dir = almalinux
 rsync_host = rsync.repo.almalinux.org
 rsync_dir = almalinux/
+
+[ceph]
+verbose = true
+sync_type = csc-sync-ceph
+frequency = tri-hourly
+local_dir = ceph
+rsync_host = download.ceph.com

merlin/sync/sync.go

@@ -55,8 +55,6 @@ func getSyncCommand(repo *config.Repo) (cmds [][]string) {
 		return append(cmds, cscSyncDebianStep1(repo), cscSyncDebianStep2(repo))
 	case "csc-sync-debian-cd":
 		return append(cmds, cscSyncDebianCD(repo))
-	case "csc-sync-gentoo":
-		return append(cmds, cscSyncGentoo(repo))
 	case "csc-sync-s3":
 		return append(cmds, cscSyncS3(repo))
 	case "csc-sync-ssh":
@@ -216,7 +214,7 @@ func cscSyncCDImage(repo *config.Repo) []string {
 
 func cscSyncCephStep1(repo *config.Repo) []string {
 	args := []string{
-		"rsync", "--stats", "--progress",
+		"nice", "rsync", "--stats", "--progress",
 		repo.RsyncHost + "::ceph",
 		"--recursive", "--times", "--links",
 		"--hard-links",
@@ -236,7 +234,7 @@ func cscSyncCephStep1(repo *config.Repo) []string {
 
 func cscSyncCephStep2(repo *config.Repo) []string {
 	args := []string{
-		"&&", "rsync", "--stats", "--progress",
+		"nice", "rsync", "--stats", "--progress",
 		repo.RsyncHost + "::ceph",
 		"--recursive", "--times", "--links",
 		"--hard-links", "--delete-after",
@@ -291,24 +289,17 @@ func cscSyncDebianCD(repo *config.Repo) []string {
 	return args
 }
 
-func cscSyncGentoo(repo *config.Repo) []string {
-	repo.RsyncUser = "gentoo"
-	repo.PasswordFile = "gentoo-distfiles"
-	return cscSyncStandard(repo)
-}
-
-// TODO: check for special stuff that rcloning S3 needs
 func cscSyncS3(repo *config.Repo) []string {
+	// could not find a argument for "RCLONE_CONFIG_S3_TYPE=s3"
+	// so an adhoc solution was put in spawnProcess (if possible remove)
 	args := []string{
-		// RsyncHost is just a regular host (https://s3.repo.saltproject.io) in this case
-		"RCLONE_CONFIG_S3_ENDPOINT=" + repo.RsyncHost,
-		"RCLONE_CONFIG_S3_TYPE=s3",
-		"RCLONE_CONFIG_S3_PROVIDER=Other",
-		"RCLONE_CONFIG_S3_ENV_AUTH=false",
 		"nice", "rclone", "sync",
 		"--fast-list",
 		"--use-server-modtime",
-		"--bind", config.Conf.IPv4Address,
+		"--bind=" + config.Conf.IPv4Address,
+		"--s3-endpoint=" + repo.RsyncHost, // RCLONE_CONFIG_S3_ENDPOINT=$2
+		"--s3-provider=Other",             // RCLONE_CONFIG_S3_PROVIDER=Other
+		"--s3-env-auth=false",             // RCLONE_CONFIG_S3_ENV_AUTH=false
 		"s3:s3/",
 	}
 	args = append(args, buildDownloadDir(repo))
@@ -322,8 +313,7 @@ func cscSyncSSH(repo *config.Repo) []string {
 		"--stats", "-4",
 	}
 	args = append(args, addConditionalFlags(repo, stdFlags|delete)...)
-	// PasswordFile should point to the SSH_KEYFILE
+	args = append(args, "-e", fmt.Sprintf("\"ssh -b %s -i %s\"", config.Conf.IPv4Address, repo.PasswordFile))
-	args = append(args, "-e", fmt.Sprintf("'ssh -b %s -i %s'", config.Conf.IPv4Address, repo.PasswordFile))
 	args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo))
 
 	return args
@@ -372,12 +362,10 @@ func cscSyncWget(repo *config.Repo) []string {
 		"--no-host-directories",
 		"--cut-dirs=1",
 		"--content-disposition",
-		"--execute",
+		"--execute", "robots=off",
-		"robots=off",
 		"--recursive",
 		"--reject", "\"*\\?*\"",
 		"--directory-prefix=" + buildDownloadDir(repo),
-		// RsyncHost is just a regular host (https://mirror.racket-lang.org/installers/) in this case
 		repo.RsyncHost,
 	}
 

merlin/sync/utils.go

@@ -28,6 +28,14 @@ func spawnProcess(repo *config.Repo, args []string) (ch <-chan *exec.Cmd) {
 	}
 
 	cmd := exec.Command(args[0], args[1:]...)
+
+	// This appears to be the only way to set env variables for exec.Command
+	// The other env varaibles for rclone s3 were converted to arguments
+	// but I was unable to find a way to convert this one.
+	if repo.SyncType == "csc-sync-s3" {
+		cmd.Env = append(cmd.Env, "RCLONE_CONFIG_S3_TYPE=s3")
+	}
+
 	repo.Logger.Debug("Starting process")
 
 	if err := cmd.Start(); err != nil {